Merge branch 'v1.0.5-misc-fixes' into 'v1.0.5'

Misc hot fixes

See merge request !596

nwchem @ a7b28a77

-Subproject commit a7b28a77c171d71d33319fefd36c6a838c2ac89d

octopus @ c81f95fc

-Subproject commit c81f95fc41677153e83534bde23f97d9ada0b477

onetep @ 69b6ffa7

-Subproject commit 69b6ffa7a641d8272908b4165f8a2f6544de4ff9

openkim @ c6347e23

-Subproject commit c6347e239415efbc41113aa1724ade634f9d7853

orca @ 18f33663

-Subproject commit 18f33663ad81d6b8be0acc96c0157e0a99c8aefa

phonopy @ 447dff39

-Subproject commit 447dff397c257516506f46ce87426ec233a72ae7

psi4 @ 9a505802

-Subproject commit 9a505802dec0a15c35018acf06e1dab4f23f519a

qbox @ 11c986c9

-Subproject commit 11c986c9f684368dea281fa22b9f4525e79a0ca4

quantum-espresso @ 8aa96aad

-Subproject commit 8aa96aadd03ecca3451a6dc8b57025999d58af1a

siesta @ 2c055bd7

-Subproject commit 2c055bd71c6b12e49d4afa856640e040bb36b3a4

tinker @ 9462067d

-Subproject commit 9462067dc98fb775dd4051c21f040d4336a32750

turbomole @ 8e3bf938

-Subproject commit 8e3bf9385ef52961a5ee9f42297f191d8c93efe3

vasp @ ff58b607

-Subproject commit ff58b6073c104e643384b786706fa33fe48b2281

wien2k @ 4ac3b4f9

-Subproject commit 4ac3b4f9fe998d36c037ce90fd907ffe5409e027

workflow @ d9d499b8

+Subproject commit d9d499b873b1bd1847a13376acc48a928661036a

yambo @ a435ec10

-Subproject commit a435ec10d4ed9d723bcd37cc198ea54c21034b49

gui/src/components/dataset/ResolveDOI.js

@@ -44,7 +44,7 @@ export default function ResolveDOI() {
     })
     let { doi } = doiMatch.params
 
-    api.get('/datasets', {doi: doi})
+    api.get('/datasets/', {doi: doi})
       .then(response => {
         if (response.pagination.total >= 1) {
           const dataset_id = response.data[0].dataset_id

gui/src/components/uploads/EditMetaDataDialog.js

@@ -338,7 +338,7 @@ function EditMetaDataDialog({...props}) {
   }, [])
 
   const createNewDatasets = useCallback(() => {
-    const promises = actions.filter(action => action.create_dataset).map(action => api.post(`/datasets`, {dataset_name: action.create_dataset}))
+    const promises = actions.filter(action => action.create_dataset).map(action => api.post(`/datasets/`, {dataset_name: action.create_dataset}))
     return Promise.all(promises)
   }, [api, actions])
 

gui/src/components/uploads/UploadPage.js

@@ -355,6 +355,7 @@ function UploadPage() {
       })
       .catch((error) => {
         if (error instanceof DoesNotExist && deleteClicked) {
+          history.push(getUrl('uploads', location))
           return
         }
         if (!hasUpload && error.apiMessage) {
@@ -363,17 +364,15 @@ function UploadPage() {
           raiseError(error)
         }
       })
-  }, [api, hasUpload, uploadId, pagination, deleteClicked, raiseError, setData, setApiData])
+  }, [api, hasUpload, uploadId, pagination, deleteClicked, raiseError, setData, setApiData, location, history])
 
   // constant fetching of upload data when necessary
   useEffect(() => {
     if (isProcessing) {
       const interval = setInterval(fetchData(), 1000)
       return () => clearInterval(interval)
-    } else if (deleteClicked) {
-      history.push(getUrl('uploads', location))
     }
-  }, [fetchData, isProcessing, deleteClicked, history, location])
+  }, [fetchData, isProcessing])
 
   // initial fetching of upload data
   useEffect(fetchData(), [fetchData])

nomad/cli/dev.py

@@ -156,16 +156,14 @@ def parser_metadata():
     import yaml
     import os
     import os.path
+    from glob import glob
 
     parsers_metadata = {}
-    parsers_path = 'dependencies/parsers'
-    for parser_dir in os.listdir(parsers_path):
-        parser_path = os.path.join(parsers_path, parser_dir)
-        parser_metadata_file = os.path.join(parser_path, 'metadata.yaml')
-        if os.path.exists(parser_metadata_file):
-            with open(parser_metadata_file) as f:
-                parser_metadata = yaml.load(f, Loader=yaml.FullLoader)
-            parsers_metadata[parser_dir] = parser_metadata
+    parsers_path = './dependencies/parsers'
+    for parser_metadata_file in sorted(glob(f'{parsers_path}/**/metadata.yaml', recursive=True)):
+        with open(parser_metadata_file) as f:
+            parser_metadata = yaml.load(f, Loader=yaml.FullLoader)
+        parsers_metadata[os.path.basename(os.path.dirname(parser_metadata_file))] = parser_metadata
 
     parsers_metadata = {
         key: parsers_metadata[key]
@@ -216,66 +214,89 @@ def update_parser_readmes(parser):
     generic_fn = './README.parsers.md'
     parser_path = './dependencies/parsers/'
 
-    for num, ddir in enumerate(sorted(glob(parser_path + '*/')), start=1):
-        if parser is not None and parser != ddir.split(os.sep)[-2]:
-            print(f'Skip {ddir}')
-            continue
-
-        _, parser_dirname = os.path.split(ddir)
-        print('{} Working on {}' .format(num, parser_dirname))
+    # Open general template
+    with open(generic_fn, 'r') as generic:  # read only
+        generic_contents = generic.read()
 
-        # Open general template
-        with open(generic_fn, 'r') as generic:  # read only
-            body = generic.read()
+    # Replace the comment at the top of the gereral template
+    generic_contents = re.sub(
+        rf'\*\*\*Note:\*\* This is a general README file for NOMAD parsers, '
+        rf'consult the README of specific parser projects for more detailed '
+        rf'information!\*\n\n', '', generic_contents)
 
-        # Open local YAML metadata file
-        local_mdata = ddir + 'metadata.yaml'
-        with open(local_mdata, 'r') as mdata_f:
-            mdata = yaml.load(mdata_f, Loader=yaml.FullLoader)
-            if mdata.get('codeName', '').strip() == '':
-                mdata['codeName'] = os.path.basename(os.path.dirname(ddir))
-            if 'preamble' not in mdata:
-                mdata['preamble'] = ''
+    def open_metadata(path):
+        # read local yaml metadata file
+        with open(path, 'r') as metadata_f:
+            try:
+                metadata = yaml.load(metadata_f, Loader=yaml.FullLoader)
+            except Exception as e:
+                print(f'Error reading metadata.yaml: {e}')
+                metadata = None
+        return metadata
+
+    def replace(metadata, contents, path):
+        # replace placelder in contents with metadata values
+        for key in metadata.keys():
+            replace = metadata.get(key)
+            if replace is None:
+                continue
+            print(f'\tReplacing {key} with {replace}')
+            contents = re.sub(rf'\${key}\$', replace, contents)
+
+        # save file
+        with open(path, 'w') as f:
+            f.write(contents.strip())
+            f.write('\n')
+
+    for local_readme in sorted(glob(f'{parser_path}/*/{local_fn}')):
+        parser_dir = os.path.dirname(local_readme)
+        project_name = os.path.basename(parser_dir)
+        print(f'Working on {parser_dir}')
+
+        contents = generic_contents
+        # if metadata file under the parser directory exists, it is a single parser project
+        single = os.path.isfile(os.path.join(parser_dir, 'metadata.yaml'))
+
+        if single:
+            metadata = open_metadata(os.path.join(parser_dir, 'metadata.yaml'))
+            # git path is given by nomad-parser-codename
+            metadata['gitPath'] = f'nomad-parser-{project_name}'
+            parser_header, parser_specs = '', ''
+        else:
+            # replace header for the single parser with that for a group of parsers
+            parser_header_re = r'(\nThis is a NOMAD parser[\s\S]+?Archive format\.\n)'
+            parser_header = re.search(parser_header_re, contents).group(1)
+            group_header = 'This is a collection of the NOMAD parsers for the following $codeName$ codes:\n\n$parserList$'
+            contents = re.sub(parser_header_re, group_header, contents)
+            # remove individual parser specs
+            parser_specs_re = r'(For \$codeLabel\$ please provide[\s\S]+?\$tableOfFiles\$)\n\n'
+            parser_specs = re.search(parser_specs_re, contents).group(1)
+            contents = re.sub(parser_specs_re, '', contents)
+            metadata = dict(
+                gitPath=f'{project_name}-parsers',
+                parserGitUrl=f'https://github.com/nomad-coe/{project_name}-parsers.git',
+                parserSpecific='')
+        if metadata.get('codeName', '').strip() == '':
+            metadata['codeName'] = project_name
+        if 'preamble' not in metadata:
+            metadata['preamble'] = ''
+
+        # if this is a group of parser, find all individdual parsers and write the
+        # parser specs
+        parser_list = ''
+        for index, local_metadata in enumerate(sorted(glob(f'{parser_dir}/*/*/metadata.yaml'))):
+            metadata_parser = open_metadata(local_metadata)
+            # contents is simply the parser header and specs
+            contents_parser = f'{parser_header}\n{parser_specs}'
+            replace(metadata_parser, contents_parser, os.path.join(os.path.dirname(local_metadata), local_fn))
+            # add the codename to the list of parsers for the group header
+            codelabel = metadata_parser.get('codeLabel', os.path.basename(os.path.dirname(local_metadata)))
+            codeurl = metadata_parser.get('codeUrl', '')
+            parser_list = rf'{parser_list}{index + 1}. [{codelabel}]({codeurl})\n'
+        metadata['parserList'] = parser_list.strip()
 
         # Find & Replace Parser`s metadata on its README file
-        local_readme = ddir + local_fn
-        with open(local_readme, 'w') as local:
-            for key in mdata.keys():
-                ignore = ['codeLabelStyle', 'parserDirName']
-                if key in ignore:
-                    continue
-
-                find = r'\$' + key + r'\$'
-                replace = mdata[key]
-                print(f'\tReplacing {key} with {replace}')
-
-                if key == 'parserSpecific':
-                    if mdata[key] != '':
-                        replace = r'## Parser Specific\n' + replace
-
-                body = re.sub(find, replace, body)
-
-            # Extra: to replace the codeName (there's no YAML key for this)
-            key = 'codeName'
-            print('\tReplacing', key)
-            find = r'\$' + key + r'\$'
-            replace = parser_dirname
-            body = re.sub(find, replace, body)
-
-            # Extra: to replace the comment at the top of the gereral template
-            find = (
-                r'\*\*\*Note:\*\* This is a general README file for NOMAD parsers, '
-                r'consult the README of specific parser projects for more detailed '
-                r'information!\*\n\n')
-            replace = ''
-            print('\tReplacing the top comment')
-            # print('\t', find, ' -> ', replace)
-            body = re.sub(find, replace, body)
-
-            # save file
-            local.seek(0)  # go to the top
-            local.write(body.strip())
-            local.truncate()
+        replace(metadata, contents, local_readme)
 
 
 @dev.command(help='Adds a few pieces of data to NOMAD.')