Converted the photoemission skeleton in a general skeleton parser.

.gitignore

@@ -53,7 +53,7 @@ lib/
 env/
 
 # Setuptools install folder
-photoemissionparser.egg-info/
+*.egg-info/
 
 .directory
 *.annotate

README.md

-# parser-photoemission
+# parser-skeleton
+
+## About
+
+This is not a real parser, its a skeleton for parsers. To write you own parsers, its
+best to fork this skeleton and use it as a template.
 
 ## Setup and run example
 
-We are currently targeting Python 3.6
+We are currently targeting Python 3.6. Some nomad dependencies might still have problems
+with 3.7++. It will definitely not work with 2.x.
 
 Best use a virtual environment:
 ```
@@ -12,20 +18,44 @@ source .pyenv/bin/activate
 
 Clone and install the nomad infrastructure and the necessary dependencies (including this parser)
 ```
-git clone https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR --branch photoemission nomad
+git clone https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR nomad
 git submodule update --init
 pip install -r requirements.txt
 ./dependencies.sh -e
 ```
 
+Fork the this project, e.g. [gitlab](https://gitlab.mpcdf.mpg.de/nomad-lab/parser-skeleton).
+Rename your fork in its settings/advanced and move it to the nomad-lab namespace.
+
+Add your parser to the nomad project on a separate branch:
+```
+git checkout -b your-parser-name
+git submodule add https://gitlab.mpcdf.mpg.de/nomad-lab/parser-your-parser-name dependencies/parsers/your-parser-name
+```
+
+Do the necessary changes:
+- [setup.py](setup.py): Change the project metadata
+- [skeletonparser](skeletonparser): Change the directory name
+- [skeletonparser/__init__.py](skeletonparser/__init__.py): Implement your parser, change the class names
+- [skeletonparser/__main__.py](skeletonparser/__main__.py): Change the module/class names
+- [skeletonparser/skeleton.nomadmetainfo.json](skeletonparser/skeleton.nomadmetainfo.json): Change the name, add your metadata definitions.
+- [README.md](README.md): Change this readme accordingly.
+
+
 To run the parser:
 ```
-cd nomad/dependencies/parser/photoemission
-python -m photoemissionparser <test_file>
+cd nomad/dependencies/parsers/your-parsername
+python -m your-parser-pythonpackage <test_file>
 ```
 
 ## Docs
 
 [metainfo](https://metainfo.nomad-coe.eu/nomadmetainfo_public/archive.html)
 
-[nomad@fairdi, tmp.](http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/docs)
+[nomad@fairdi, tmp.](http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/latest/docs)
+
+## FAQ
+
+For any questions, please open issues (regarding parser development and using this skeleton)
+in this [parser-skeleton project](https://gitlab.mpcdf.mpg.de/nomad-lab/parser-skeleton/issues).
+We will compile a FAQ from your issues.

photoemissionparser/photoemission.nomadmetainfo.json

-{
-    "type": "nomad_meta_info_1_0",
-    "description": "Public experimental material science meta info, not specific to any experimental method",
-    "metaInfos": [ {
-        "description": "Root section that contains all information about an experiment",
-        "kindStr": "type_section",
-        "name": "section_root",
-        "superNames": []
-      }, {
-        "description": "Contains information relating to an archive.",
-        "name": "experiment_location",
-        "dtypeStr": "C",
-        "superNames": ["section_root"]
-      }
-    ]
-  }

setup.py

@@ -16,10 +16,10 @@ from setuptools import setup, find_packages
 
 def main():
     setup(
-        name='photoemissionparser',
+        name='skeletonparser',  # replace with new name for parser's python package
         version='0.1',
-        description='NOMAD parser implementation for photoemission experiment files.',
-        author='',
+        description='A skeleton NOMAD parser implementation.',  # change accordingly
+        author='',  # add your names
         license='APACHE 2.0',
         packages=find_packages(),
         install_requires=[

photoemissionparser/__init__.py → skeletonparser/__init__.py

-# Copyright 2016-2018 R. Patrick Xian
+# Copyright 2016-2018 Markus Scheidgen
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
 #   You may obtain a copy of the License at
@@ -13,6 +13,10 @@
 
 import sys
 import os.path
+import json
+import ase
+import numpy as np
+from datetime import datetime
 
 from nomadcore.simple_parser import SimpleMatcher
 from nomadcore.baseclasses import ParserInterface, AbstractBaseParser
@@ -20,16 +24,21 @@ from nomadcore.baseclasses import ParserInterface, AbstractBaseParser
 from nomad.parsing import LocalBackend
 
 
-class PhotoemissionParserInterface(ParserInterface):
+class SkeletonParserInterface(ParserInterface):
 
     def get_metainfo_filename(self):
-        """ The parser specific metainfo. This file must be part of the nomad-meta-info. """
-        return os.path.join(os.path.dirname(__file__), 'photoemission.nomadmetainfo.json')
+        """
+        The parser specific metainfo. To include other metadata definitions, use
+        the 'dependencies' key to refer to other local nomadmetainfo.json files or
+        to nomadmetainfo.json files that are part of the general nomad-meta-info
+        submodule (i.e. ``dependencies/nomad-meta-info``).
+         """
+        return os.path.join(os.path.dirname(__file__), 'skeleton.nomadmetainfo.json')
 
     def get_parser_info(self):
         """ Basic info about parser used in archive data and logs. """
         return {
-            'name': 'vaspoutcar_parser',
+            'name': 'you parser name',
             'version': '1.0.0'
         }
 
@@ -39,13 +48,42 @@ class PhotoemissionParserInterface(ParserInterface):
 
     def setup_main_parser(self, _):
         """ Setup the actual parser (behind this interface) """
-        self.main_parser = PhotoemissionParser(self.parser_context)
+        self.main_parser = SkeletonParser(self.parser_context)
 
 
-class PhotoemissionParser(AbstractBaseParser):
+class SkeletonParser(AbstractBaseParser):
     def parse(self, filepath):
         backend = self.parser_context.super_backend
 
-        root_gid = backend.openSection('section_root')
-        backend.addValue('experiment_location', 'here...')
-        backend.closeSection('section_root', root_gid)
+        with open(filepath, 'rt') as f:
+            data = json.load(f)
+
+        # You need to open sections before you can add values or sub sections to it.
+        # The returned 'gid' can be used to reference a specific section if multiple
+        # sections of the same type are opened.
+        root_gid = backend.openSection('section_experiment')
+        # Values are added to the open section of the given metadata definitions. In
+        # the following case 'experiment_location' is a quantity of 'section_experiment'.
+        # When multiple sections of the same type (e.g. 'section_experiment') are open,
+        # you can use the 'gid' as an additional argument.
+        backend.addValue('experiment_location', data.get('location'))
+        # Values do not necessarely have to be read from the parsed file.
+        backend.addValue('experiment_method_name', data.get('method', 'Bare eyes'))
+        # The backend will check the type of the given value agains the metadata definition.
+        backend.addValue('experiment_time', int(datetime.strptime(data.get('date'), '%d.%M.%Y').timestamp()))
+
+        # Subsections work like before. The parent section must still be open.
+        sample_gid = backend.openSection('section_sample')
+        backend.addValue('sample_chemical_name', data.get('sample_chemical'))
+        backend.addValue('sample_chemical_formula', data.get('sample_formula'))
+        backend.addValue('sample_temperature', data.get('sample_temp'))
+
+        atoms = set(ase.Atoms(data.get('sample_formula')).get_chemical_symbols())
+        # To add arrays (vectors, matrices, etc.) use addArrayValues and provide a
+        # numpy array. The shape of the numpy array must match the shape defined in
+        # the respective metadata definition.
+        backend.addArrayValues('sample_atom_labels', np.array(list(atoms)))
+
+        # Close sections in the reverse order.
+        backend.closeSection('section_sample', sample_gid)
+        backend.closeSection('section_experiment', root_gid)

photoemissionparser/__main__.py → skeletonparser/__main__.py

-# Copyright 2016-2018 R. Patrick Xian
+# Copyright 2016-2018 Markus Scheidgen
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
 #   You may obtain a copy of the License at
@@ -14,13 +14,13 @@
 import sys
 
 from nomad.parsing import LocalBackend
-from photoemissionparser import PhotoemissionParserInterface
+from skeletonparser import SkeletonParserInterface
 
 if __name__ == "__main__":
     # instantiate the parser via its interface with a LocalBackend
-    parser = PhotoemissionParserInterface(backend=LocalBackend)
+    parser = SkeletonParserInterface(backend=LocalBackend)
     # call the actual parsing with the given mainfile
     parser.parse(sys.argv[1])
     # print the results stored in the LocalBackend
     parser.parser_context.super_backend.write_json(
-        sys.stdout, pretty=True, root_sections=['section_root'])
+        sys.stdout, pretty=True, root_sections=['section_experiment'])

skeletonparser/skeleton.nomadmetainfo.json

+{
+  "type": "nomad_meta_info_1_0",
+  "description": "Parser specific metadata definitions.",
+  "dependencies":[
+    {
+      "metainfoPath":"general.nomadmetainfo.json"
+    },
+    {
+      "metainfoPath":"general.experimental.nomadmetainfo.json"
+    }
+  ],
+  "metaInfos": [
+    {
+      "description": "Contains information relating to an archive.",
+      "name": "experiment_location",
+      "dtypeStr": "C",
+      "superNames": ["section_experiment"]
+    }
+  ]
+}

tests/example.metadata.json

+{
+    "type":"skeleton experimental metadata format 1.0",
+    "date":"24.12.2018",
+    "location":"Northpole",
+    "sample_formula":"H2O",
+    "sample_chemical":"Ice",
+    "sample_state":"Frozen",
+    "result":"https://bitcoinist.com/wp-content/uploads/2018/08/shutterstock_764225425.jpg",
+    "sample_temp":384
+}
\ No newline at end of file

tests/generate_test_data.py

+# Copyright 2016-2018 Markus Scheidgen
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import ase.data
+import random
+import time
+import zipfile
+import json
+import sys
+
+
+if __name__ == '__main__':
+
+    count = int(sys.argv[1])
+    zipfile_name = sys.argv[2]
+
+    atoms = ase.data.chemical_symbols
+    low_numbers_for_atoms = [1, 1, 2, 2, 2, 2, 2, 3, 3, 4]
+
+    methods = ['human eye examination', 'optical microscopes', 'TEM', 'photonemission', 'spectroscopy']
+    locations = ['FHI', 'HU Berlin', 'Desy', 'ACME']
+    chemical_names = ['super gue', 'dilithium', 'unoptanium', 'graphene 2.0', 'ducktape']
+
+    now = int(time.time())
+
+    for i in range(0, count + 1):
+        with zipfile.ZipFile(zipfile_name, 'w') as zf:
+            formula = []
+            for _ in range(0, random.choice(low_numbers_for_atoms)):
+                formula.append('%s%d' % (random.choice(atoms), random.choice(low_numbers_for_atoms)))
+            data = dict(
+                type='skeleton experimental metadata format 1.0',
+                location=random.choice(locations),
+                method=random.choice(methods),
+                date=time.strftime('%d.%M.%Y', time.localtime(random.uniform(0, now))),
+                sample_chemical=random.choice(chemical_names),
+                sample_formula=''.join(formula),
+                sample_temp=random.uniform(0, 6000)
+            )
+
+            with zf.open('experiment_%d/data.json' % i, 'w') as f:
+                f.write(bytes(json.dumps(data, indent=4), 'utf-8'))