Merged latest v0.6.2 changes.

gui/src/components/App.js

@@ -198,8 +198,14 @@ class NavigationUnstyled extends React.Component {
   }
 
   componentDidMount() {
-    fetch(`${guiBase}/meta.json`)
-      .then((response) => response.json())
+    fetch(`${guiBase}/meta.json`, {
+      method: 'GET',
+      cache: 'no-cache',
+      headers: {
+        'Pragma': 'no-cache',
+        'Cache-Control': 'no-cache, no-store'
+      }
+    }).then((response) => response.json())
       .then((meta) => {
         if (meta.version !== packageJson.version) {
           console.log('GUI API version mismatch')

nomad/app/api/raw.py

@@ -366,7 +366,7 @@ class RawFileQueryResource(Resource):
             abort(400, message='bad parameter types')
 
         search_request = search.SearchRequest()
-        add_query(search_request)
+        add_query(search_request, search_request_parser)
 
         calcs = sorted([
             (entry['upload_id'], entry['mainfile'])

nomad/app/api/repo.py

@@ -102,7 +102,7 @@ def add_common_parameters(request_parser):
     for quantity in search.quantities.values():
         request_parser.add_argument(
             quantity.name, help=quantity.description,
-            action='append' if quantity.multi else None)
+            action=quantity.argparse_action if quantity.multi else None)
 
 
 repo_request_parser = pagination_request_parser.copy()
@@ -129,14 +129,16 @@ search_request_parser = api.parser()
 add_common_parameters(search_request_parser)
 
 
-def add_query(search_request: search.SearchRequest):
+def add_query(search_request: search.SearchRequest, parser=repo_request_parser):
     """
     Help that adds query relevant request parameters to the given SearchRequest.
     """
+    args = {key: value for key, value in parser.parse_args().items() if value is not None}
+
     # owner
     try:
         search_request.owner(
-            request.args.get('owner', 'all'),
+            args.get('owner', 'all'),
             g.user.user_id if g.user is not None else None)
     except ValueError as e:
         abort(401, getattr(e, 'message', 'Invalid owner parameter'))
@@ -144,8 +146,8 @@ def add_query(search_request: search.SearchRequest):
         abort(400, getattr(e, 'message', 'Invalid owner parameter'))
 
     # time range
-    from_time_str = request.args.get('from_time', None)
-    until_time_str = request.args.get('until_time', None)
+    from_time_str = args.get('from_time', None)
+    until_time_str = args.get('until_time', None)
 
     try:
         from_time = rfc3339DateTime.parse(from_time_str) if from_time_str is not None else None
@@ -156,7 +158,7 @@ def add_query(search_request: search.SearchRequest):
 
     # optimade
     try:
-        optimade = request.args.get('optimade', None)
+        optimade = args.get('optimade', None)
         if optimade is not None:
             q = filterparser.parse_filter(optimade)
             search_request.query(q)
@@ -165,8 +167,7 @@ def add_query(search_request: search.SearchRequest):
 
     # search parameter
     search_request.search_parameters(**{
-        key: request.args.getlist(key) if search.quantities[key] else request.args.get(key)
-        for key in request.args.keys()
+        key: value for key, value in args.items()
         if key not in ['optimade'] and key in search.quantities})
 
 
@@ -210,7 +211,7 @@ class RepoCalcsResource(Resource):
         """
 
         search_request = search.SearchRequest()
-        add_query(search_request)
+        add_query(search_request, repo_request_parser)
 
         try:
             scroll = bool(request.args.get('scroll', False))
@@ -325,7 +326,7 @@ class RepoQuantityResource(Resource):
         """
 
         search_request = search.SearchRequest()
-        add_query(search_request)
+        add_query(search_request, repo_quantity_search_request_parser)
 
         try:
             after = request.args.get('after', None)

nomad/app/api/upload.py

@@ -222,6 +222,7 @@ class UploadListResource(Resource):
             pagination=dict(total=total, page=page, per_page=per_page),
             results=results), 200
 
+    @api.doc(security=list(api.authorizations.keys()))  # weird bug, this should not be necessary
     @api.doc('upload')
     @api.expect(upload_metadata_parser)
     @api.response(400, 'To many uploads')

nomad/config.py

@@ -187,7 +187,7 @@ client = NomadConfig(
     url='http://localhost:8000/fairdi/nomad/latest/api'
 )
 
-version = '0.6.0'
+version = '0.7.0'
 commit = gitinfo.commit
 release = 'devel'
 domain = 'DFT'

nomad/datamodel/base.py

@@ -260,6 +260,7 @@ class DomainQuantity:
         elastic_field: An optional elasticsearch key. Default is the name of the quantity.
         elastic_value: A collable that takes a :class:`CalcWithMetadata` as input and produces the
             value for the elastic search index.
+        argparse_action: Action to use on argparse, either append or split for multi values. Append is default.
     """
 
     def __init__(
@@ -268,7 +269,8 @@ class DomainQuantity:
             zero_aggs: bool = True, metadata_field: str = None,
             elastic_mapping: type = None,
             elastic_search_type: str = 'term', elastic_field: str = None,
-            elastic_value: Callable[[Any], Any] = None):
+            elastic_value: Callable[[Any], Any] = None,
+            argparse_action: str = 'append'):
 
         self._name: str = None
         self.description = description
@@ -281,6 +283,7 @@ class DomainQuantity:
         self.elastic_search_type = elastic_search_type
         self.metadata_field = metadata_field
         self.elastic_field = elastic_field
+        self.argparse_action = argparse_action
 
         self.elastic_value = elastic_value
         if self.elastic_value is None:
@@ -353,7 +356,9 @@ class Domain:
         pid=DomainQuantity(description='Search for the pid.'),
         raw_id=DomainQuantity(description='Search for the raw_id.'),
         mainfile=DomainQuantity(description='Search for the mainfile.'),
-        external_id=DomainQuantity(description='External user provided id. Does not have to be unique necessarily.'),
+        external_id=DomainQuantity(
+            description='External user provided id. Does not have to be unique necessarily.',
+            multi=True, argparse_action='split', elastic_search_type='terms'),
         dataset=DomainQuantity(
             elastic_field='datasets.name', multi=True, elastic_search_type='match',
             description='Search for a particular dataset by name.'),

nomad/parsing/__init__.py

@@ -71,10 +71,11 @@ based on NOMAD-coe's *python-common* module.
     :members:
 """
 
-from typing import Callable, IO, Union
+from typing import Callable, IO, Union, Dict
 import magic
 import gzip
 import bz2
+import lzma
 import os.path
 
 from nomad import files, config
@@ -87,7 +88,8 @@ from nomad.parsing.artificial import TemplateParser, GenerateRandomParser, Chaos
 
 _compressions = {
     b'\x1f\x8b\x08': ('gz', gzip.open),
-    b'\x42\x5a\x68': ('bz2', bz2.open)
+    b'\x42\x5a\x68': ('bz2', bz2.open),
+    b'\xfd\x37\x7a': ('xz', lzma.open)
 }
 
 
@@ -116,7 +118,7 @@ def match_parser(mainfile: str, upload_files: Union[str, files.StagingUploadFile
     with open(mainfile_path, 'rb') as f:
         compression, open_compressed = _compressions.get(f.read(3), (None, open))
 
-    with open_compressed(mainfile_path, 'rb') as cf:
+    with open_compressed(mainfile_path, 'rb') as cf:  # type: ignore
         buffer = cf.read(config.parser_matching_size)
 
     mime_type = magic.from_buffer(buffer, mime=True)
@@ -147,14 +149,14 @@ parsers = [
     LegacyParser(
         name='parsers/vasp', code_name='VASP',
         parser_class_name='vaspparser.VASPRunParserInterface',
-        mainfile_mime_re=r'(application/xml)|(text/.*)',
+        mainfile_mime_re=r'(application/.*)|(text/.*)',
         mainfile_contents_re=(
             r'^\s*<\?xml version="1\.0" encoding="ISO-8859-1"\?>\s*'
             r'?\s*<modeling>'
             r'?\s*<generator>'
             r'?\s*<i name="program" type="string">\s*vasp\s*</i>'
             r'?'),
-        supported_compressions=['gz', 'bz2']
+        supported_compressions=['gz', 'bz2', 'xz']
     ),
     VaspOutcarParser(
         name='parsers/vasp-outcar', code_name='VASP',

nomad/search.py

@@ -319,6 +319,13 @@ class SearchRequest:
 
         value = quantity.elastic_value(value)
 
+        if quantity.elastic_search_type == 'terms':
+            if not isinstance(value, list):
+                value = [value]
+            self.q &= Q('terms', **{quantity.elastic_field: value})
+
+            return self
+
         if isinstance(value, list):
             values = value
         else:

ops/helm/nomad/Chart.yaml

 apiVersion: v1
-appVersion: "0.6.0"
+appVersion: "0.7.0"
 description: A Helm chart for Kubernetes that only runs nomad services and uses externally hosted databases.
 name: nomad
-version: 0.6.0
+version: 0.7.0

ops/helm/nomad/templates/gui-deployment.yml

@@ -39,6 +39,16 @@ data:
         rewrite ^{{ .Values.proxy.external.path }}/gui/service-worker.js /nomad/service-worker.js break;
       }
 
+      location {{ .Values.proxy.external.path }}/gui/meta.json {
+        add_header Last-Modified $date_gmt;
+        add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
+        if_modified_since off;
+        expires off;
+        etag off;
+        root      /app/;
+        rewrite ^{{ .Values.proxy.external.path }}/gui/meta.json /nomad/meta.json break;
+      }
+
       location {{ .Values.proxy.external.path }}/api/uploads {
         client_max_body_size 35g;
         proxy_request_buffering off;

setup.py

@@ -12,7 +12,7 @@ reqs = [str(ir.req) for ir in install_reqs if 'sphinxcontrib.httpdomain' not in
 
 setup(
     name='nomad',
-    version='0.6.0',
+    version='0.7.0',
     description='The nomad@FAIRDI infrastructure python package',
     py_modules=['nomad'],
     install_requires=reqs,

tests/app/test_api.py

@@ -600,19 +600,19 @@ class TestRepo():
         calc_with_metadata.update(
             calc_id='2', uploader=other_test_user.user_id, published=True,
             with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
-            external_id='external_id')
+            external_id='external_2')
         calc_with_metadata.update(
             atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
         search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
 
         calc_with_metadata.update(
             calc_id='3', uploader=other_test_user.user_id, published=False,
-            with_embargo=False, pid=3, external_id='external_id')
+            with_embargo=False, pid=3, external_id='external_3')
         search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
 
         calc_with_metadata.update(
             calc_id='4', uploader=other_test_user.user_id, published=True,
-            with_embargo=True, pid=4, external_id='external_id')
+            with_embargo=True, pid=4, external_id='external_4')
         search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
 
     def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
@@ -711,30 +711,35 @@ class TestRepo():
         rv = api.get('/repo/%s' % query_string)
         self.assert_search(rv, calcs)
 
-    @pytest.mark.parametrize('calcs, quantity, value', [
-        (2, 'system', 'bulk'),
-        (0, 'system', 'atom'),
-        (1, 'atoms', 'Br'),
-        (1, 'atoms', 'Fe'),
-        (0, 'atoms', ['Fe', 'Br', 'A', 'B']),
-        (0, 'only_atoms', ['Br', 'Si']),
-        (1, 'only_atoms', ['Fe']),
-        (1, 'only_atoms', ['Br', 'K', 'Si']),
-        (1, 'only_atoms', ['Br', 'Si', 'K']),
-        (1, 'comment', 'specific'),
-        (1, 'authors', 'Leonard Hofstadter'),
-        (2, 'files', 'test/mainfile.txt'),
-        (2, 'paths', 'mainfile.txt'),
-        (2, 'paths', 'test'),
-        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number']),
-        (0, 'quantities', 'dos'),
-        (1, 'external_id', 'external_id'),
-        (0, 'external_id', 'external')
+    @pytest.mark.parametrize('calcs, quantity, value, user', [
+        (2, 'system', 'bulk', 'test_user'),
+        (0, 'system', 'atom', 'test_user'),
+        (1, 'atoms', 'Br', 'test_user'),
+        (1, 'atoms', 'Fe', 'test_user'),
+        (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
+        (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
+        (1, 'only_atoms', ['Fe'], 'test_user'),
+        (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
+        (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
+        (1, 'comment', 'specific', 'test_user'),
+        (1, 'authors', 'Hofstadter, Leonard', 'test_user'),
+        (2, 'files', 'test/mainfile.txt', 'test_user'),
+        (2, 'paths', 'mainfile.txt', 'test_user'),
+        (2, 'paths', 'test', 'test_user'),
+        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
+        (0, 'quantities', 'dos', 'test_user'),
+        (2, 'external_id', 'external_2,external_3', 'other_test_user'),
+        (1, 'external_id', 'external_2', 'test_user'),
+        (1, 'external_id', 'external_2,external_3', 'test_user'),
+        (0, 'external_id', 'external_x', 'test_user')
     ])
-    def test_search_parameters(self, api, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
+    def test_search_parameters(
+            self, api, example_elastic_calcs, no_warn, test_user_auth,
+            other_test_user_auth, calcs, quantity, value, user):
+        user_auth = test_user_auth if user == 'test_user' else other_test_user_auth
         query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
 
-        rv = api.get('/repo/?%s' % query_string, headers=test_user_auth)
+        rv = api.get('/repo/?%s' % query_string, headers=user_auth)
         logger.debug('run search quantities test', query_string=query_string)
         data = self.assert_search(rv, calcs)