diff --git a/gui/src/components/App.js b/gui/src/components/App.js
index f234f4bf00e74b0484da435357a58d66b903951d..1b37809e508819a78b83bb499506e38e31d600cf 100644
--- a/gui/src/components/App.js
+++ b/gui/src/components/App.js
@@ -198,8 +198,14 @@ class NavigationUnstyled extends React.Component {
}
componentDidMount() {
- fetch(`${guiBase}/meta.json`)
- .then((response) => response.json())
+ fetch(`${guiBase}/meta.json`, {
+ method: 'GET',
+ cache: 'no-cache',
+ headers: {
+ 'Pragma': 'no-cache',
+ 'Cache-Control': 'no-cache, no-store'
+ }
+ }).then((response) => response.json())
.then((meta) => {
if (meta.version !== packageJson.version) {
console.log('GUI API version mismatch')
diff --git a/nomad/app/api/raw.py b/nomad/app/api/raw.py
index 9736303b14ed2f02f29f455dc3ade49470f92ef4..2afcf637ee0720aac16665148c540918aab8460b 100644
--- a/nomad/app/api/raw.py
+++ b/nomad/app/api/raw.py
@@ -366,7 +366,7 @@ class RawFileQueryResource(Resource):
abort(400, message='bad parameter types')
search_request = search.SearchRequest()
- add_query(search_request)
+ add_query(search_request, search_request_parser)
calcs = sorted([
(entry['upload_id'], entry['mainfile'])
diff --git a/nomad/app/api/repo.py b/nomad/app/api/repo.py
index 95a64bb35e7ce1483d963ed5b253b2ab412196e4..b0a148179e65e2b476f55db7e86605638f57e858 100644
--- a/nomad/app/api/repo.py
+++ b/nomad/app/api/repo.py
@@ -102,7 +102,7 @@ def add_common_parameters(request_parser):
for quantity in search.quantities.values():
request_parser.add_argument(
quantity.name, help=quantity.description,
- action='append' if quantity.multi else None)
+ action=quantity.argparse_action if quantity.multi else None)
repo_request_parser = pagination_request_parser.copy()
@@ -129,14 +129,16 @@ search_request_parser = api.parser()
add_common_parameters(search_request_parser)
-def add_query(search_request: search.SearchRequest):
+def add_query(search_request: search.SearchRequest, parser=repo_request_parser):
"""
Help that adds query relevant request parameters to the given SearchRequest.
"""
+ args = {key: value for key, value in parser.parse_args().items() if value is not None}
+
# owner
try:
search_request.owner(
- request.args.get('owner', 'all'),
+ args.get('owner', 'all'),
g.user.user_id if g.user is not None else None)
except ValueError as e:
abort(401, getattr(e, 'message', 'Invalid owner parameter'))
@@ -144,8 +146,8 @@ def add_query(search_request: search.SearchRequest):
abort(400, getattr(e, 'message', 'Invalid owner parameter'))
# time range
- from_time_str = request.args.get('from_time', None)
- until_time_str = request.args.get('until_time', None)
+ from_time_str = args.get('from_time', None)
+ until_time_str = args.get('until_time', None)
try:
from_time = rfc3339DateTime.parse(from_time_str) if from_time_str is not None else None
@@ -156,7 +158,7 @@ def add_query(search_request: search.SearchRequest):
# optimade
try:
- optimade = request.args.get('optimade', None)
+ optimade = args.get('optimade', None)
if optimade is not None:
q = filterparser.parse_filter(optimade)
search_request.query(q)
@@ -165,8 +167,7 @@ def add_query(search_request: search.SearchRequest):
# search parameter
search_request.search_parameters(**{
- key: request.args.getlist(key) if search.quantities[key] else request.args.get(key)
- for key in request.args.keys()
+ key: value for key, value in args.items()
if key not in ['optimade'] and key in search.quantities})
@@ -210,7 +211,7 @@ class RepoCalcsResource(Resource):
"""
search_request = search.SearchRequest()
- add_query(search_request)
+ add_query(search_request, repo_request_parser)
try:
scroll = bool(request.args.get('scroll', False))
@@ -325,7 +326,7 @@ class RepoQuantityResource(Resource):
"""
search_request = search.SearchRequest()
- add_query(search_request)
+ add_query(search_request, repo_quantity_search_request_parser)
try:
after = request.args.get('after', None)
diff --git a/nomad/app/api/upload.py b/nomad/app/api/upload.py
index 649e4517ccb6e9c68fb2caf986a117f01769b89a..2e2f8f687483286fbc1678266bbff8c9bfbba076 100644
--- a/nomad/app/api/upload.py
+++ b/nomad/app/api/upload.py
@@ -222,6 +222,7 @@ class UploadListResource(Resource):
pagination=dict(total=total, page=page, per_page=per_page),
results=results), 200
+ @api.doc(security=list(api.authorizations.keys())) # weird bug, this should not be necessary
@api.doc('upload')
@api.expect(upload_metadata_parser)
@api.response(400, 'To many uploads')
diff --git a/nomad/config.py b/nomad/config.py
index 35af6acc8475ff3e3e6b9674d1784cfbefe38891..3ee162f5a2e421a013eaa6db1265082925d0ad7e 100644
--- a/nomad/config.py
+++ b/nomad/config.py
@@ -187,7 +187,7 @@ client = NomadConfig(
url='http://localhost:8000/fairdi/nomad/latest/api'
)
-version = '0.6.0'
+version = '0.7.0'
commit = gitinfo.commit
release = 'devel'
domain = 'DFT'
diff --git a/nomad/datamodel/base.py b/nomad/datamodel/base.py
index b2f602c3967256fc4c4cda331a4f99c13089f084..1cd190c2b0822edd542c0f475b385fccb478bc19 100644
--- a/nomad/datamodel/base.py
+++ b/nomad/datamodel/base.py
@@ -260,6 +260,7 @@ class DomainQuantity:
elastic_field: An optional elasticsearch key. Default is the name of the quantity.
elastic_value: A collable that takes a :class:`CalcWithMetadata` as input and produces the
value for the elastic search index.
+ argparse_action: Action to use on argparse, either append or split for multi values. Append is default.
"""
def __init__(
@@ -268,7 +269,8 @@ class DomainQuantity:
zero_aggs: bool = True, metadata_field: str = None,
elastic_mapping: type = None,
elastic_search_type: str = 'term', elastic_field: str = None,
- elastic_value: Callable[[Any], Any] = None):
+ elastic_value: Callable[[Any], Any] = None,
+ argparse_action: str = 'append'):
self._name: str = None
self.description = description
@@ -281,6 +283,7 @@ class DomainQuantity:
self.elastic_search_type = elastic_search_type
self.metadata_field = metadata_field
self.elastic_field = elastic_field
+ self.argparse_action = argparse_action
self.elastic_value = elastic_value
if self.elastic_value is None:
@@ -353,7 +356,9 @@ class Domain:
pid=DomainQuantity(description='Search for the pid.'),
raw_id=DomainQuantity(description='Search for the raw_id.'),
mainfile=DomainQuantity(description='Search for the mainfile.'),
- external_id=DomainQuantity(description='External user provided id. Does not have to be unique necessarily.'),
+ external_id=DomainQuantity(
+ description='External user provided id. Does not have to be unique necessarily.',
+ multi=True, argparse_action='split', elastic_search_type='terms'),
dataset=DomainQuantity(
elastic_field='datasets.name', multi=True, elastic_search_type='match',
description='Search for a particular dataset by name.'),
diff --git a/nomad/parsing/__init__.py b/nomad/parsing/__init__.py
index 4375ac1aa55542e1b1497491ceb4f64abc844bb2..0481965589583002be8ac4c0b959361357b4ac5a 100644
--- a/nomad/parsing/__init__.py
+++ b/nomad/parsing/__init__.py
@@ -71,10 +71,11 @@ based on NOMAD-coe's *python-common* module.
:members:
"""
-from typing import Callable, IO, Union
+from typing import Callable, IO, Union, Dict
import magic
import gzip
import bz2
+import lzma
import os.path
from nomad import files, config
@@ -87,7 +88,8 @@ from nomad.parsing.artificial import TemplateParser, GenerateRandomParser, Chaos
_compressions = {
b'\x1f\x8b\x08': ('gz', gzip.open),
- b'\x42\x5a\x68': ('bz2', bz2.open)
+ b'\x42\x5a\x68': ('bz2', bz2.open),
+ b'\xfd\x37\x7a': ('xz', lzma.open)
}
@@ -116,7 +118,7 @@ def match_parser(mainfile: str, upload_files: Union[str, files.StagingUploadFile
with open(mainfile_path, 'rb') as f:
compression, open_compressed = _compressions.get(f.read(3), (None, open))
- with open_compressed(mainfile_path, 'rb') as cf:
+ with open_compressed(mainfile_path, 'rb') as cf: # type: ignore
buffer = cf.read(config.parser_matching_size)
mime_type = magic.from_buffer(buffer, mime=True)
@@ -147,14 +149,14 @@ parsers = [
LegacyParser(
name='parsers/vasp', code_name='VASP',
parser_class_name='vaspparser.VASPRunParserInterface',
- mainfile_mime_re=r'(application/xml)|(text/.*)',
+ mainfile_mime_re=r'(application/.*)|(text/.*)',
mainfile_contents_re=(
r'^\s*<\?xml version="1\.0" encoding="ISO-8859-1"\?>\s*'
r'?\s*<modeling>'
r'?\s*<generator>'
r'?\s*<i name="program" type="string">\s*vasp\s*</i>'
r'?'),
- supported_compressions=['gz', 'bz2']
+ supported_compressions=['gz', 'bz2', 'xz']
),
VaspOutcarParser(
name='parsers/vasp-outcar', code_name='VASP',
diff --git a/nomad/search.py b/nomad/search.py
index 9a85add85b111602c668292dcfcf553e7c297163..5e9f4a9aee30b41841d34060058786daa5fc4127 100644
--- a/nomad/search.py
+++ b/nomad/search.py
@@ -319,6 +319,13 @@ class SearchRequest:
value = quantity.elastic_value(value)
+ if quantity.elastic_search_type == 'terms':
+ if not isinstance(value, list):
+ value = [value]
+ self.q &= Q('terms', **{quantity.elastic_field: value})
+
+ return self
+
if isinstance(value, list):
values = value
else:
diff --git a/ops/helm/nomad/Chart.yaml b/ops/helm/nomad/Chart.yaml
index 787b7894f5acc792e2a839d0b16d7688dba45eac..ee30ccb632c53cd7c521425500ca5681f75e14ec 100644
--- a/ops/helm/nomad/Chart.yaml
+++ b/ops/helm/nomad/Chart.yaml
@@ -1,5 +1,5 @@
apiVersion: v1
-appVersion: "0.6.0"
+appVersion: "0.7.0"
description: A Helm chart for Kubernetes that only runs nomad services and uses externally hosted databases.
name: nomad
-version: 0.6.0
+version: 0.7.0
diff --git a/ops/helm/nomad/templates/gui-deployment.yml b/ops/helm/nomad/templates/gui-deployment.yml
index b7eff504f4d55c99463c591bff631b7d0f7094da..3c18d78dac167e8a718d3b5bdfd79d6a88b6bcd6 100644
--- a/ops/helm/nomad/templates/gui-deployment.yml
+++ b/ops/helm/nomad/templates/gui-deployment.yml
@@ -39,6 +39,16 @@ data:
rewrite ^{{ .Values.proxy.external.path }}/gui/service-worker.js /nomad/service-worker.js break;
}
+ location {{ .Values.proxy.external.path }}/gui/meta.json {
+ add_header Last-Modified $date_gmt;
+ add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
+ if_modified_since off;
+ expires off;
+ etag off;
+ root /app/;
+ rewrite ^{{ .Values.proxy.external.path }}/gui/meta.json /nomad/meta.json break;
+ }
+
location {{ .Values.proxy.external.path }}/api/uploads {
client_max_body_size 35g;
proxy_request_buffering off;
diff --git a/setup.py b/setup.py
index 3b733e4bd72422fe3a735548bf7c7e921e04e2cf..f151c837a5795a71030c088acc46165fbaa9e437 100644
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,7 @@ reqs = [str(ir.req) for ir in install_reqs if 'sphinxcontrib.httpdomain' not in
setup(
name='nomad',
- version='0.6.0',
+ version='0.7.0',
description='The nomad@FAIRDI infrastructure python package',
py_modules=['nomad'],
install_requires=reqs,
diff --git a/tests/app/test_api.py b/tests/app/test_api.py
index 7960221992062e28baa7d69d12d0b49ed15d5a11..fc5516db179cdf20f48c7935489f9a5d2e9b3861 100644
--- a/tests/app/test_api.py
+++ b/tests/app/test_api.py
@@ -600,19 +600,19 @@ class TestRepo():
calc_with_metadata.update(
calc_id='2', uploader=other_test_user.user_id, published=True,
with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
- external_id='external_id')
+ external_id='external_2')
calc_with_metadata.update(
atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
calc_with_metadata.update(
calc_id='3', uploader=other_test_user.user_id, published=False,
- with_embargo=False, pid=3, external_id='external_id')
+ with_embargo=False, pid=3, external_id='external_3')
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
calc_with_metadata.update(
calc_id='4', uploader=other_test_user.user_id, published=True,
- with_embargo=True, pid=4, external_id='external_id')
+ with_embargo=True, pid=4, external_id='external_4')
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
@@ -711,30 +711,35 @@ class TestRepo():
rv = api.get('/repo/%s' % query_string)
self.assert_search(rv, calcs)
- @pytest.mark.parametrize('calcs, quantity, value', [
- (2, 'system', 'bulk'),
- (0, 'system', 'atom'),
- (1, 'atoms', 'Br'),
- (1, 'atoms', 'Fe'),
- (0, 'atoms', ['Fe', 'Br', 'A', 'B']),
- (0, 'only_atoms', ['Br', 'Si']),
- (1, 'only_atoms', ['Fe']),
- (1, 'only_atoms', ['Br', 'K', 'Si']),
- (1, 'only_atoms', ['Br', 'Si', 'K']),
- (1, 'comment', 'specific'),
- (1, 'authors', 'Leonard Hofstadter'),
- (2, 'files', 'test/mainfile.txt'),
- (2, 'paths', 'mainfile.txt'),
- (2, 'paths', 'test'),
- (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number']),
- (0, 'quantities', 'dos'),
- (1, 'external_id', 'external_id'),
- (0, 'external_id', 'external')
+ @pytest.mark.parametrize('calcs, quantity, value, user', [
+ (2, 'system', 'bulk', 'test_user'),
+ (0, 'system', 'atom', 'test_user'),
+ (1, 'atoms', 'Br', 'test_user'),
+ (1, 'atoms', 'Fe', 'test_user'),
+ (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
+ (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
+ (1, 'only_atoms', ['Fe'], 'test_user'),
+ (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
+ (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
+ (1, 'comment', 'specific', 'test_user'),
+ (1, 'authors', 'Hofstadter, Leonard', 'test_user'),
+ (2, 'files', 'test/mainfile.txt', 'test_user'),
+ (2, 'paths', 'mainfile.txt', 'test_user'),
+ (2, 'paths', 'test', 'test_user'),
+ (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
+ (0, 'quantities', 'dos', 'test_user'),
+ (2, 'external_id', 'external_2,external_3', 'other_test_user'),
+ (1, 'external_id', 'external_2', 'test_user'),
+ (1, 'external_id', 'external_2,external_3', 'test_user'),
+ (0, 'external_id', 'external_x', 'test_user')
])
- def test_search_parameters(self, api, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
+ def test_search_parameters(
+ self, api, example_elastic_calcs, no_warn, test_user_auth,
+ other_test_user_auth, calcs, quantity, value, user):
+ user_auth = test_user_auth if user == 'test_user' else other_test_user_auth
query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
- rv = api.get('/repo/?%s' % query_string, headers=test_user_auth)
+ rv = api.get('/repo/?%s' % query_string, headers=user_auth)
logger.debug('run search quantities test', query_string=query_string)
data = self.assert_search(rv, calcs)