From 66b372db0c6b5cbb9b72aa0ea462181468f11987 Mon Sep 17 00:00:00 2001 From: Fawzi Mohamed Date: Mon, 19 Feb 2018 15:34:54 +0100 Subject: [PATCH] automatically set calculation_file_uri --- common/python/nomadcore/simple_parser.py | 49 ++++++++++++++++++++---- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/common/python/nomadcore/simple_parser.py b/common/python/nomadcore/simple_parser.py index 4d116dc..8834a60 100644 --- a/common/python/nomadcore/simple_parser.py +++ b/common/python/nomadcore/simple_parser.py @@ -20,6 +20,9 @@ from nomadcore.unit_conversion import unit_conversion from nomadcore.caching_backend import CachingLevel, ActiveBackend from nomadcore.annotator import Annotator import io +from future.standard_library import install_aliases +install_aliases() +from urllib.parse import urlparse, urlunparse logger = logging.getLogger("nomadcore.simple_parser") annotate = False @@ -59,6 +62,16 @@ class PushbackLineFile(object): self.lines.append(line) self.lineNr -= 1 +def uriFromRelPath(baseUri, basePath, newPath): + """returns an uri corresponding to newPath assuming that base path has uri baseUri. +This will never change the net location (archive).""" + p1 = os.path.normpath(os.path.abspath(basePath)) + p2 = os.path.normpath(os.path.abspath(newPath)) + rPath = os.path.relpath(p2,p1) + bUri = urlparse(baseUri) + nUri = bUri._replace(path = os.path.normpath(os.path.join(bUri.path, rPath))) + return urlunparse(nUri) + class SimpleMatcher(object): """A Something that matches either a single line, or multiple lines grouped together. This is the base of a declarative parser.""" @@ -1071,8 +1084,8 @@ class SimpleParserBuilder(object): logger.info("Parsing tree after optimization:") self.rootMatcher.printParsingTree() - def buildParser(self, fIn, backend, superContext): - return SimpleParser(self, fIn, backend, superContext) + def buildParser(self, fIn, backend, superContext, baseUri, basePath): + return SimpleParser(self, fIn, backend, superContext, baseUri, basePath) def writeMatchers(self, outF, extraIndent = 0): outF.write("[") @@ -1102,7 +1115,7 @@ class SimpleParserBuilder(object): outF.write("}") class SimpleParser(object): - def __init__(self, parserBuilder, fIn, backend, superContext = None): + def __init__(self, parserBuilder, fIn, backend, superContext = None, baseUri = None, basePath = None): self.parserBuilder = parserBuilder self.fIn = fIn self.backend = backend @@ -1110,6 +1123,13 @@ class SimpleParser(object): self.skipped = 0 self.superContext = superContext self.lastMatch = {} + self.baseUri = baseUri + self.basePath = basePath + if backend.sectionManagers is not None: + r=backend.sectionManagers.get("section_run") + if r: + r.onOpen.append(self.emitBaseUri) + annofilename = None # by default, ignore empty lines in coverage analysis self.coverageIgnore = getattr(superContext, 'coverageIgnore', @@ -1128,6 +1148,17 @@ class SimpleParser(object): for k,v in sorted(compiledRootMatcher.matcher.fixedStartValues.items()): compiledRootMatcher.addValue(backend, k, v) + def uriForPath(self, path): + """return the uri corresponding to the given path""" + if self.baseUri and self.basePath and path: + return uriFromRelPath(self.baseUri, self.basePath, path) + return path + + def emitBaseUri(self, backend, gIndex, section): + """writes out the uri to the sources""" + if self.baseUri: + backend.addValue("calculation_file_uri", self.baseUri) + def enterInState(self, stateIndex): compiledMatcher = self.parserBuilder.compiledMatchers[stateIndex] sects = OrderedDict() @@ -1285,9 +1316,9 @@ def compileParser(simpleParser, metaInfo, metaInfoToKeep, default_units=None, me logger.debug(s.getvalue()) return parserBuilder -def runParser(compiledParser, backend, superContext, fIn): +def runParser(compiledParser, backend, superContext, fIn, uri, path): """parses the open file fIn with the given compiledParser into the backend using superContext as parser SuperContext""" - parser = compiledParser.buildParser(PushbackLineFile(fIn), backend, superContext = superContext) + parser = compiledParser.buildParser(PushbackLineFile(fIn), backend, superContext = superContext, baseUri = uri, basePath = path) try: superContext.startedParsing(fIn.name, parser) except AttributeError: @@ -1301,7 +1332,7 @@ def defaultParseFile(parserInfo): with open(path, "r") as fIn: backend.startedParsingSession(uri, parserInfo) try: - parsingStats = runParser(parserBuilder, backend, superContext, fIn) + parsingStats = runParser(parserBuilder, backend, superContext, fIn, uri, path) backend.finishedParsingSession( "ParseSuccess", None, parsingStats=parsingStats) @@ -1538,6 +1569,7 @@ class AncillaryParser(object): parser: The mains parser that is already running. It is used to get the current metadata and backend. superContext: Context for the ancillary parser. """ + self.mainParser = parser default_units = parser.parserBuilder.default_units metainfo_units = parser.parserBuilder.metainfo_units # compile parser @@ -1568,7 +1600,10 @@ class AncillaryParser(object): Args: fIn: File that is parsed. """ - runParser(self.compiledParser, self.backend, self.superContext, PushbackLineFile(fIn)) + currentUri = self.mainParser.uriForPath(fIn.name) + if currentUri: + self.backend.superBackend.addValue("calculation_file_uri", currentUri) + runParser(self.compiledParser, self.backend, self.superContext, PushbackLineFile(fIn), self.mainParser.baseUri, self.mainParser.basePath) class ParserOptimizer(object): -- GitLab