Merge branch 'bugfixes' into 'v0.10.4'

Bugfixes

See merge request !345

example @ 32752f63

-Subproject commit 793036f889ae94141ed5a5914d30223538d3ff52
+Subproject commit 32752f6333954855c1e216c321cb1db036f7787e

docs/parser.md

@@ -40,7 +40,7 @@ pip install -e .
 
 The main code file `exampleparser/parser.py` should look like this:
 ```python
-class ExampleParser(FairdiParser):
+class ExampleParser(MatchingParser):
     def __init__(self):
         super().__init__(name='parsers/example', code_name='EXAMPLE')
 
@@ -52,7 +52,7 @@ class ExampleParser(FairdiParser):
         run.program_name = 'EXAMPLE'
 ```
 
-A parser is a simple program with a single class in it. The base class `FairdiParser`
+A parser is a simple program with a single class in it. The base class `MatchingParser`
 provides the necessary interface to NOMAD. We provide some basic information
 about our parser in the constructor. The *main* function `run` simply takes a filepath
 and empty archive as input. Now its up to you, to open the given file and populate the
@@ -62,7 +62,7 @@ populate the archive with a *root section* `Run` and set the program name to `EX
 You can run the parser with the included `__main__.py`. It takes a file as argument and
 you can run it like this:
 ```sh
-python -m exampleparser test/data/example.out
+python -m exampleparser tests/data/example.out
 ```
 
 The output should show the log entry and the minimal archive with one `section_run` and
@@ -196,7 +196,7 @@ for calculation in mainfile_parser.get('calculation'):
 
 You can still run the parse on the given example file:
 ```sh
-python -m exampleparser test/data/example.out
+python -m exampleparser tests/data/example.out
 ```
 
 Now you should get a more comprehensive archive with all the provided information from
@@ -242,7 +242,7 @@ the output.
 def test_example():
     parser = rExampleParser()
     archive = EntryArchive()
-    parser.run('test/data/example.out', archive, logging)
+    parser.run('tests/data/example.out', archive, logging)
 
     run = archive.section_run[0]
     assert len(run.section_system) == 2
@@ -261,7 +261,7 @@ features of the underlying code/format.
 ## Structured data files with numpy
 **TODO: examples**
 
-The `DataTextFileParser` uses the numpy.loadtxt function to load an structured data file.
+The `DataTextParser` uses the numpy.loadtxt function to load an structured data file.
 The loaded data can be accessed from property *data*.
 
 ## XML Parser
@@ -275,12 +275,13 @@ data type conversion is performed, which can be switched off by setting *convert
 ## Add the parser to NOMAD
 
 NOMAD has to manage multiple parsers and during processing needs to decide what parsers
-to run on what files. To manage parser, a few more information about parsers is necessary.
+to run on what files. To decide what parser is use, NOMAD processing relies on specific
+parser attributes.
 
-Consider the example, where we use the `FairdiParser` constructor to add additional
+Consider the example, where we use the `MatchingParser` constructor to add additional
 attributes that determine for what files the parser is indented:
 ```python
-class ExampleParser(FairdiParser):
+class ExampleParser(MatchingParser):
     def __init__(self):
         super().__init__(
             name='parsers/example', code_name='EXAMPLE', code_homepage='https://www.example.eu/',
@@ -292,6 +293,10 @@ class ExampleParser(FairdiParser):
 run on files with matching mime type. The mime-type is *guessed* with libmagic.
 - `mainfile_contents_re`: A regular expression that is applied to the first 4k of a file.
 The parser is only run on files where this matches.
+- `mainfile_name_re`: A regular expression that can be used to match against the name and path of the file.
+
+Not all of these attributes have to be used. Those that are given must all match in order
+to use the parser on a file.
 
 The nomad infrastructure keep a list of parser objects (in `nomad/parsing/parsers.py::parsers`).
 These parser are considered in the order they appear in the list. The first matching parser
@@ -303,7 +308,7 @@ added to the infrastructure parser tests (`tests/parsing/test_parsing.py`).
 Once the parser is added, it become also available through the command line interface and
 normalizers are applied as well:
 ```sh
-nomad parser test/data/example.out
+nomad parser tests/data/example.out
 ```
 
 ## Developing an existing parser

gui/src/components/Markdown.js

@@ -257,28 +257,25 @@ function Markdown(props) {
     // - turn metainfo names into links into the metainfo
     let word = ''
     content = children.replace(/^ +/gm, '').split('').map((c, i) => {
-      if (c === '$') {
-        if (state[state.length - 1] === 'math') {
+      if (c === '`' || c === '$') {
+        if (state.peek === c) {
           state.pop()
         } else {
-          state.push('math')
-        }
-      } else if (c === '`' || c === '(' || c === ')') {
-        if (state[state.length - 1] === 'code') {
-          state.pop()
-        } else {
-          state.push('code')
-        }
-      } else {
-        if (state.peek === 'escape') {
-          state.pop()
+          state.push(c)
         }
       }
 
-      if (!state[state.length - 1] && c.match(/[a-zA-Z0-9_]/)) {
+      if (c === '[') {
+        state.push(c)
+      }
+      if (c === ']' && state.peek === '[') {
+        state.pop()
+      }
+
+      if (c.match(/[a-zA-Z0-9_]/)) {
         word += c
       } else {
-        if (word.match(/_/g)) {
+        if (state.length === 0 && (word.match(/_/g) || word.match(/[a-z]+[A-Z]/g)) && word.match(/^[a-zA-Z0-9_]+$/g) && c !== ']') {
           const path = metainfoPath(word)
           if (path) {
             word = `[\`${word}\`](/metainfo/${metainfoPath(word)})`

gui/src/components/archive/Browser.js

@@ -133,13 +133,12 @@ Browser.propTypes = ({
 export const laneContext = React.createContext()
 const useLaneStyles = makeStyles(theme => ({
   root: {
-    minWidth: 200,
-    maxWidth: 512,
+    width: 'min-content',
     borderRight: `solid 1px ${grey[500]}`,
-    display: 'block'
+    display: 'inline-block'
   },
   container: {
-    display: 'block',
+    display: 'inline-block',
     height: '100%',
     overflowY: 'scroll'
   },

gui/src/components/archive/metainfo.js

@@ -156,7 +156,7 @@ export function isReference(property) {
 export function path(nameOrDef) {
   let def
   if (typeof nameOrDef === 'string') {
-    def = defsByName[nameOrDef] && defsByName[nameOrDef].find(def => def.m_def !== 'SubSection')
+    def = defsByName[nameOrDef] && defsByName[nameOrDef].find(def => true)
   } else {
     def = nameOrDef
   }
@@ -165,6 +165,10 @@ export function path(nameOrDef) {
     return null
   }
 
+  if (def.m_def === 'SubSection') {
+    def = resolveRef(def.sub_section)
+  }
+
   if (def.m_def === 'Category') {
     return `${def._package.name.split('.')[0]}/category_definitions@${def._qualifiedName}`
   }

gui/src/components/uploads/UploadPage.js

@@ -353,8 +353,8 @@ class UploadPage extends React.Component {
             \`\`\`
 
             ### Form data vs. streaming
-            NOMAD accepts stream data (\`-T <local_file>\`) (like in the
-            examples above) or multi-part form data (\`-X PUT -f file=@<local_file>\`):
+            NOMAD accepts stream data \`-T <local_file>\` (like in the
+            examples above) or multi-part form data \`-X PUT -F file=@<local_file>\`:
             \`\`\`
             ${uploadCommand.upload_command_form}
             \`\`\`
@@ -363,8 +363,8 @@ class UploadPage extends React.Component {
             more information (e.g. the file name) to our servers (see below).
 
             #### Upload names
-            With multi-part form data (\`-X PUT -f file=@<local_file>\`), your upload will
-            be named after the file by default. With stream data (\`-T <local_file>\`)
+            With multi-part form data \`-X PUT -F file=@<local_file>\`, your upload will
+            be named after the file by default. With stream data \`-T <local_file>\`
             there will be no default name. To set a custom name, you can use the URL
             parameter \`name\`:
             \`\`\`

nomad/app/flask/api/upload.py

@@ -267,7 +267,7 @@ class UploadListResource(Resource):
         user = g.user
         from_oasis = oasis_upload_id is not None
         if from_oasis:
-            if not g.user.is_oasis_admin:
+            if not g.user.full_user().is_oasis_admin:
                 abort(401, 'Only an oasis admin can perform an oasis upload.')
             if oasis_uploader_id is None:
                 abort(400, 'You must provide the original uploader for an oasis upload.')
@@ -281,7 +281,7 @@ class UploadListResource(Resource):
 
         uploader_id = request.args.get('uploader_id')
         if uploader_id is not None:
-            if not g.user.is_admin:
+            if not g.user.full_user().is_admin:
                 abort(401, 'Only an admins can upload for other users.')
 
             user = datamodel.User.get(user_id=uploader_id)
@@ -615,10 +615,10 @@ class UploadCommandResource(Resource):
 
         upload_command_form = 'curl "%s" -X PUT -F file=@<local_file>' % upload_url
 
-        upload_command_with_name = 'curl "%s" -X PUT -T <local_file>' % upload_url_with_name
+        upload_command_with_name = 'curl "%s" -T <local_file>' % upload_url_with_name
 
         upload_progress_command = upload_command + ' | xargs echo'
-        upload_tar_command = 'tar -cf - <local_folder> | curl -# -H "%s" -T - | xargs echo' % upload_url
+        upload_tar_command = 'tar -cf - <local_folder> | curl "%s" -T - | xargs echo' % upload_url
 
         return dict(
             upload_url=upload_url,

nomad/cli/admin/admin.py

@@ -86,7 +86,7 @@ def __run_parallel(
 
 def __run_processing(
         uploads, parallel: int, process, label: str, reprocess_running: bool = False,
-        wait_for_tasks: bool = True):
+        wait_for_tasks: bool = True, reset_first: bool = False):
 
     def run_process(upload, logger):
         logger.info(
@@ -99,19 +99,26 @@ def __run_processing(
                 current_process=upload.current_process,
                 current_task=upload.current_task, upload_id=upload.upload_id)
             return False
-        else:
+
+        if reset_first:
             upload.reset(force=True)
-            process(upload)
-            if wait_for_tasks:
-                upload.block_until_complete(interval=.5)
-            else:
-                upload.block_until_process_complete(interval=.5)
+        elif upload.process_running:
+            tasks_status = upload.tasks_status
+            if tasks_status == proc.RUNNING:
+                tasks_status = proc.FAILURE
+            upload.reset(force=True, tasks_status=tasks_status)
+
+        process(upload)
+        if wait_for_tasks:
+            upload.block_until_complete(interval=.5)
+        else:
+            upload.block_until_process_complete(interval=.5)
 
-            if upload.tasks_status == proc.FAILURE:
-                logger.info('%s with failure' % label, upload_id=upload.upload_id)
+        if upload.tasks_status == proc.FAILURE:
+            logger.info('%s with failure' % label, upload_id=upload.upload_id)
 
-            logger.info('%s complete' % label, upload_id=upload.upload_id)
-            return True
+        logger.info('%s complete' % label, upload_id=upload.upload_id)
+        return True
 
     __run_parallel(uploads, parallel=parallel, callable=run_process, label=label)
 

nomad/cli/admin/uploads.py

@@ -374,7 +374,7 @@ def re_process(ctx, uploads, parallel: int, reprocess_running: bool):
     _, uploads = query_uploads(ctx, uploads)
     __run_processing(
         uploads, parallel, lambda upload: upload.re_process_upload(), 're-processing',
-        reprocess_running=reprocess_running)
+        reprocess_running=reprocess_running, reset_first=True)
 
 
 @uploads.command(help='Repack selected uploads.')

nomad/datamodel/datamodel.py

@@ -117,6 +117,12 @@ class User(Author):
         from nomad import infrastructure
         return infrastructure.keycloak.get_user(*args, **kwargs)  # type: ignore
 
+    def full_user(self) -> 'User':
+        ''' Returns a User object with all attributes loaded from the user management system. '''
+        from nomad import infrastructure
+        assert self.user_id is not None
+        return infrastructure.keycloak.get_user(user_id=self.user_id)  # type: ignore
+
 
 class UserReference(metainfo.Reference):
     '''

nomad/datamodel/metainfo/common_dft.py

@@ -1118,8 +1118,8 @@ class BasisSet(MSection):
     dependent to the simulated cell as a whole).
 
     Basis sets used in this section_single_configuration_calculation, belonging to either
-    class, are defined in the dedicated section: [section_basis_set_cell_dependent
-    ](section_basis_set_cell_dependent) or section_basis_set_atom_centered. The
+    class, are defined in the dedicated section: section_basis_set_cell_dependent or
+    section_basis_set_atom_centered. The
     correspondence between the basis sets listed in this section and the definition given
     in the dedicated sessions is given by the two concrete metadata:
     mapping_section_basis_set_cell_dependent and mapping_section_basis_set_atom_centered.
@@ -1961,9 +1961,8 @@ class FrameSequenceUserQuantity(MSection):
 
         Dedicated metadata monitored along a sequence of frames are created for the
         conserved energy-like quantity (frame_sequence_conserved_quantity), the kinetic
-        and potential energies ([frame_sequence_kinetic_energy and
-        frame_sequence_potential_energy](frame_sequence_kinetic_energy and
-        frame_sequence_potential_energy)), the instantaneous temperature
+        and potential energies (frame_sequence_kinetic_energy and
+        frame_sequence_potential_energy), the instantaneous temperature
         (frame_sequence_temperature) and the pressure (frame_sequence_pressure).
         ''',
         categories=[Unused],
@@ -3821,8 +3820,7 @@ class Run(MSection):
     '''
     Every section_run represents a single call of a program. What exactly is contained in
     a run depends on the run type (see for example section_method and
-    section_single_configuration_calculation) and the program (see [program_info
-    ](program_info)).
+    section_single_configuration_calculation) and the program (see ProgramInfo).
     '''
 
     m_def = Section(
@@ -7859,7 +7857,7 @@ class Topology(MSection):
         description='''
         A unique string idenfiying the force field defined in this section. Strategies to
         define it are discussed in the
-        [topology\\_force\\_field\\_name](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-meta-
+        [topology_force_field_name](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-meta-
         info/wikis/metainfo/topology-force-field-name).
         ''',
         a_legacy=LegacyDefinition(name='topology_force_field_name'))

nomad/metainfo/metainfo.py

@@ -796,7 +796,13 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
             if isinstance(attr, Property):
                 attr.name = name
                 if attr.description is not None:
-                    attr.description = inspect.cleandoc(attr.description).strip()
+                    description = inspect.cleandoc(attr.description)
+                    description = description.strip()
+                    description = re.sub(
+                        r'\(https?://[^\)]*\)',
+                        lambda m: re.sub(r'\n', '', m.group(0)),
+                        description)
+                    attr.description = description
                     attr.__doc__ = attr.description
 
                 if isinstance(attr, Quantity):

nomad/processing/base.py

@@ -212,13 +212,16 @@ class Proc(Document, metaclass=ProcMetaclass):
 
         return self
 
-    def reset(self, worker_hostname: str = None, force: bool = False):
+    def reset(
+            self, worker_hostname: str = None, force: bool = False,
+            tasks_status: str = PENDING):
+
         ''' Resets the task chain. Assumes there no current running process. '''
         assert not self.process_running or force
 
         self.current_task = None
         self.process_status = None
-        self.tasks_status = PENDING
+        self.tasks_status = tasks_status
         self.errors = []
         self.warnings = []
         self.worker_hostname = worker_hostname

tests/processing/test_data.py

@@ -519,6 +519,9 @@ def test_re_pack(published: Upload, monkeypatch, with_failure):
         with upload_files.read_archive(calc.calc_id) as archive:
             archive[calc.calc_id].to_dict()
 
+    published.reload()
+    assert published.tasks_status == SUCCESS
+
 
 def mock_failure(cls, task, monkeypatch):
     def mock(self):

tests/test_cli.py

@@ -27,6 +27,7 @@ from nomad import search, processing as proc, files
 from nomad.cli import cli
 from nomad.cli.cli import POPO
 from nomad.processing import Upload, Calc
+from nomad.processing.base import SUCCESS
 
 from tests.app.flask.test_app import BlueprintClient
 from tests.app.flask.conftest import (  # pylint: disable=unused-import
@@ -253,6 +254,9 @@ class TestAdminUploads:
             with upload_files.read_archive(calc.calc_id) as archive:
                 assert calc.calc_id in archive
 
+        published.reload()
+        assert published.tasks_status == SUCCESS
+
     def test_chown(self, published, test_user, other_test_user):
         upload_id = published.upload_id
         calc = Calc.objects(upload_id=upload_id).first()