Processing now stores results to archive.

nomad/files.py

@@ -38,7 +38,7 @@ Uploads
 .. autoclass:: Upload
 
 """
-from typing import Callable, List, Any, Generator, IO, TextIO
+from typing import Callable, List, Any, Generator, IO, TextIO, cast
 import sys
 import os
 from os.path import join
@@ -268,10 +268,10 @@ class Upload():
 
 
 @contextmanager
-def write_archive_json(archive_id) -> Generator[IO, None, None]:
+def write_archive_json(archive_id) -> Generator[TextIO, None, None]:
     """ Context manager that yiels a file-like to write the archive json. """
     binary_out = io.BytesIO()
-    gzip_wrapper = gzip.open(binary_out, 'wt')
+    gzip_wrapper = cast(TextIO, gzip.open(binary_out, 'wt'))
 
     try:
         yield gzip_wrapper

nomad/parsing.py

@@ -414,18 +414,16 @@ class Parser():
 
         return False
 
-    def run(self, mainfile: str, out: TextIO=None) -> Tuple[str, List[str]]:
+    def run(self, mainfile: str) -> LocalBackend:
         """
         Runs the parser on the given mainfile. It uses :class:`LocalBackend` as
         a backend. The meta-info access is handled by the underlying NOMAD-coe parser.
 
         Args:
             mainfile: A path to a mainfile that this parser can parse.
-            out: Optional file like that is used to write the 'archive'.json results
-                 to.
 
         Returns:
-            The parser status from the backend, see :property:`LocalBackend.status`.
+            The used :class:`LocalBackend` with status information and result data.
         """
         def create_backend(meta_info):
             return LocalBackend(meta_info, debug=False)
@@ -438,10 +436,7 @@ class Parser():
         parser.parse(mainfile)
 
         backend = parser.parser_context.super_backend
-        if out is not None:
-            backend.write_json(out, pretty=True)
-
-        return backend.status
+        return backend
 
     def __repr__(self):
         return self.python_git.__repr__()

nomad/processing.py

@@ -41,6 +41,7 @@ import json
 
 import nomad.config as config
 from nomad.files import Upload, UploadError
+from nomad import files, utils
 from nomad.parsing import parsers, parser_dict
 
 # The legacy nomad code uses a logger called 'nomad'. We do not want that this
@@ -110,6 +111,7 @@ class UploadProcessing():
 
         self.parse_specs: List[Tuple[str, str]] = None
         self.parse_results: List[bool] = None
+        self.upload_hash: str = None
 
         self.status: str = 'PENDING'
         self.task_name: str = None
@@ -246,6 +248,12 @@ def open_upload(task: Task, processing: UploadProcessing) -> UploadProcessing:
         logger.debug('Could not open upload %s: %s' % (processing.upload_id, e))
         return processing.fail(e)
 
+    try:
+        processing.upload_hash = upload.hash()
+    except UploadError as e:
+        logger.error('Could not create an upload hash %s: %s' % (processing.upload_id, e))
+        return processing.fail(e)
+
     try:
         processing.parse_specs = list()
         for filename in upload.filelist:
@@ -284,18 +292,20 @@ def distributed_parse(
         chord([])(close_upload.clone(args=(processing,)))
         return processing
 
-    parses = group(parse.s(parse_spec) for parse_spec in processing.parse_specs)
+    parses = group(parse.s(processing, parse_spec) for parse_spec in processing.parse_specs)
     chord(parses)(close_upload.clone(args=(processing,)))
     return processing
 
 
 @app.task()
-def parse(parse_spec: Tuple[str, str]) -> Any:
+def parse(processing: UploadProcessing, parse_spec: Tuple[str, str]) -> Any:
+    assert processing.upload_hash is not None
+
     parser, mainfile = parse_spec
 
     logger.debug('Start %s for %s.' % parse_spec)
     try:
-        parser_dict[parser].run(mainfile)
+        parser_backend = parser_dict[parser].run(mainfile)
     except ValueError as e:
         logger.warning('%s stopped on %s: %s' % (parser, mainfile, e))
         return e
@@ -303,4 +313,10 @@ def parse(parse_spec: Tuple[str, str]) -> Any:
         logger.warning('%s stopped on %s: %s' % (parser, mainfile, e), exc_info=e)
         return e
 
-    return True  # TODO some other value?
+    archive_id = '%s/%s' % (processing.upload_hash, utils.hash(mainfile))
+    logger.debug('Written results of %s for %s to %s.' % (parser, mainfile, archive_id))
+
+    with files.write_archive_json(archive_id) as out:
+        parser_backend.write_json(out, pretty=True)
+
+    return parser_backend.status

nomad/utils.py

+from typing import Union, IO, cast
+import hashlib
+import base64
+
+
+def hash(obj: Union[IO, str]) -> str:
+    """ First 28 character of an URL safe base 64 encoded sha512 digest. """
+    hash = hashlib.sha512()
+    if getattr(obj, 'read', None) is not None:
+        for data in iter(lambda: cast(IO, obj).read(65536), b''):
+            hash.update(data)
+    elif isinstance(obj, str):
+        hash.update(obj.encode('utf-8'))
+
+    return base64.b64encode(hash.digest(), altchars=b'-_')[0:28].decode('utf-8')

tests/test_parsing.py

@@ -64,7 +64,8 @@ def test_stream_generator(pretty):
 def test_vasp_parser():
     vasp_parser = parser_dict['parsers/vasp']
     example_mainfile = '.dependencies/parsers/vasp/test/examples/xml/perovskite.xml'
-    status, errors = vasp_parser.run(example_mainfile)
+    parser_backend = vasp_parser.run(example_mainfile)
+    status, errors = parser_backend.status
 
     assert status == 'ParseSuccess'
     assert errors is None or len(errors) == 0