Commit f1f6d0e1 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merged latest v0.6.2 changes.

parents 81f52793 c8aed118
...@@ -198,8 +198,14 @@ class NavigationUnstyled extends React.Component { ...@@ -198,8 +198,14 @@ class NavigationUnstyled extends React.Component {
} }
componentDidMount() { componentDidMount() {
fetch(`${guiBase}/meta.json`) fetch(`${guiBase}/meta.json`, {
.then((response) => response.json()) method: 'GET',
cache: 'no-cache',
headers: {
'Pragma': 'no-cache',
'Cache-Control': 'no-cache, no-store'
}
}).then((response) => response.json())
.then((meta) => { .then((meta) => {
if (meta.version !== packageJson.version) { if (meta.version !== packageJson.version) {
console.log('GUI API version mismatch') console.log('GUI API version mismatch')
......
...@@ -366,7 +366,7 @@ class RawFileQueryResource(Resource): ...@@ -366,7 +366,7 @@ class RawFileQueryResource(Resource):
abort(400, message='bad parameter types') abort(400, message='bad parameter types')
search_request = search.SearchRequest() search_request = search.SearchRequest()
add_query(search_request) add_query(search_request, search_request_parser)
calcs = sorted([ calcs = sorted([
(entry['upload_id'], entry['mainfile']) (entry['upload_id'], entry['mainfile'])
......
...@@ -102,7 +102,7 @@ def add_common_parameters(request_parser): ...@@ -102,7 +102,7 @@ def add_common_parameters(request_parser):
for quantity in search.quantities.values(): for quantity in search.quantities.values():
request_parser.add_argument( request_parser.add_argument(
quantity.name, help=quantity.description, quantity.name, help=quantity.description,
action='append' if quantity.multi else None) action=quantity.argparse_action if quantity.multi else None)
repo_request_parser = pagination_request_parser.copy() repo_request_parser = pagination_request_parser.copy()
...@@ -129,14 +129,16 @@ search_request_parser = api.parser() ...@@ -129,14 +129,16 @@ search_request_parser = api.parser()
add_common_parameters(search_request_parser) add_common_parameters(search_request_parser)
def add_query(search_request: search.SearchRequest): def add_query(search_request: search.SearchRequest, parser=repo_request_parser):
""" """
Help that adds query relevant request parameters to the given SearchRequest. Help that adds query relevant request parameters to the given SearchRequest.
""" """
args = {key: value for key, value in parser.parse_args().items() if value is not None}
# owner # owner
try: try:
search_request.owner( search_request.owner(
request.args.get('owner', 'all'), args.get('owner', 'all'),
g.user.user_id if g.user is not None else None) g.user.user_id if g.user is not None else None)
except ValueError as e: except ValueError as e:
abort(401, getattr(e, 'message', 'Invalid owner parameter')) abort(401, getattr(e, 'message', 'Invalid owner parameter'))
...@@ -144,8 +146,8 @@ def add_query(search_request: search.SearchRequest): ...@@ -144,8 +146,8 @@ def add_query(search_request: search.SearchRequest):
abort(400, getattr(e, 'message', 'Invalid owner parameter')) abort(400, getattr(e, 'message', 'Invalid owner parameter'))
# time range # time range
from_time_str = request.args.get('from_time', None) from_time_str = args.get('from_time', None)
until_time_str = request.args.get('until_time', None) until_time_str = args.get('until_time', None)
try: try:
from_time = rfc3339DateTime.parse(from_time_str) if from_time_str is not None else None from_time = rfc3339DateTime.parse(from_time_str) if from_time_str is not None else None
...@@ -156,7 +158,7 @@ def add_query(search_request: search.SearchRequest): ...@@ -156,7 +158,7 @@ def add_query(search_request: search.SearchRequest):
# optimade # optimade
try: try:
optimade = request.args.get('optimade', None) optimade = args.get('optimade', None)
if optimade is not None: if optimade is not None:
q = filterparser.parse_filter(optimade) q = filterparser.parse_filter(optimade)
search_request.query(q) search_request.query(q)
...@@ -165,8 +167,7 @@ def add_query(search_request: search.SearchRequest): ...@@ -165,8 +167,7 @@ def add_query(search_request: search.SearchRequest):
# search parameter # search parameter
search_request.search_parameters(**{ search_request.search_parameters(**{
key: request.args.getlist(key) if search.quantities[key] else request.args.get(key) key: value for key, value in args.items()
for key in request.args.keys()
if key not in ['optimade'] and key in search.quantities}) if key not in ['optimade'] and key in search.quantities})
...@@ -210,7 +211,7 @@ class RepoCalcsResource(Resource): ...@@ -210,7 +211,7 @@ class RepoCalcsResource(Resource):
""" """
search_request = search.SearchRequest() search_request = search.SearchRequest()
add_query(search_request) add_query(search_request, repo_request_parser)
try: try:
scroll = bool(request.args.get('scroll', False)) scroll = bool(request.args.get('scroll', False))
...@@ -325,7 +326,7 @@ class RepoQuantityResource(Resource): ...@@ -325,7 +326,7 @@ class RepoQuantityResource(Resource):
""" """
search_request = search.SearchRequest() search_request = search.SearchRequest()
add_query(search_request) add_query(search_request, repo_quantity_search_request_parser)
try: try:
after = request.args.get('after', None) after = request.args.get('after', None)
......
...@@ -222,6 +222,7 @@ class UploadListResource(Resource): ...@@ -222,6 +222,7 @@ class UploadListResource(Resource):
pagination=dict(total=total, page=page, per_page=per_page), pagination=dict(total=total, page=page, per_page=per_page),
results=results), 200 results=results), 200
@api.doc(security=list(api.authorizations.keys())) # weird bug, this should not be necessary
@api.doc('upload') @api.doc('upload')
@api.expect(upload_metadata_parser) @api.expect(upload_metadata_parser)
@api.response(400, 'To many uploads') @api.response(400, 'To many uploads')
......
...@@ -187,7 +187,7 @@ client = NomadConfig( ...@@ -187,7 +187,7 @@ client = NomadConfig(
url='http://localhost:8000/fairdi/nomad/latest/api' url='http://localhost:8000/fairdi/nomad/latest/api'
) )
version = '0.6.0' version = '0.7.0'
commit = gitinfo.commit commit = gitinfo.commit
release = 'devel' release = 'devel'
domain = 'DFT' domain = 'DFT'
......
...@@ -260,6 +260,7 @@ class DomainQuantity: ...@@ -260,6 +260,7 @@ class DomainQuantity:
elastic_field: An optional elasticsearch key. Default is the name of the quantity. elastic_field: An optional elasticsearch key. Default is the name of the quantity.
elastic_value: A collable that takes a :class:`CalcWithMetadata` as input and produces the elastic_value: A collable that takes a :class:`CalcWithMetadata` as input and produces the
value for the elastic search index. value for the elastic search index.
argparse_action: Action to use on argparse, either append or split for multi values. Append is default.
""" """
def __init__( def __init__(
...@@ -268,7 +269,8 @@ class DomainQuantity: ...@@ -268,7 +269,8 @@ class DomainQuantity:
zero_aggs: bool = True, metadata_field: str = None, zero_aggs: bool = True, metadata_field: str = None,
elastic_mapping: type = None, elastic_mapping: type = None,
elastic_search_type: str = 'term', elastic_field: str = None, elastic_search_type: str = 'term', elastic_field: str = None,
elastic_value: Callable[[Any], Any] = None): elastic_value: Callable[[Any], Any] = None,
argparse_action: str = 'append'):
self._name: str = None self._name: str = None
self.description = description self.description = description
...@@ -281,6 +283,7 @@ class DomainQuantity: ...@@ -281,6 +283,7 @@ class DomainQuantity:
self.elastic_search_type = elastic_search_type self.elastic_search_type = elastic_search_type
self.metadata_field = metadata_field self.metadata_field = metadata_field
self.elastic_field = elastic_field self.elastic_field = elastic_field
self.argparse_action = argparse_action
self.elastic_value = elastic_value self.elastic_value = elastic_value
if self.elastic_value is None: if self.elastic_value is None:
...@@ -353,7 +356,9 @@ class Domain: ...@@ -353,7 +356,9 @@ class Domain:
pid=DomainQuantity(description='Search for the pid.'), pid=DomainQuantity(description='Search for the pid.'),
raw_id=DomainQuantity(description='Search for the raw_id.'), raw_id=DomainQuantity(description='Search for the raw_id.'),
mainfile=DomainQuantity(description='Search for the mainfile.'), mainfile=DomainQuantity(description='Search for the mainfile.'),
external_id=DomainQuantity(description='External user provided id. Does not have to be unique necessarily.'), external_id=DomainQuantity(
description='External user provided id. Does not have to be unique necessarily.',
multi=True, argparse_action='split', elastic_search_type='terms'),
dataset=DomainQuantity( dataset=DomainQuantity(
elastic_field='datasets.name', multi=True, elastic_search_type='match', elastic_field='datasets.name', multi=True, elastic_search_type='match',
description='Search for a particular dataset by name.'), description='Search for a particular dataset by name.'),
......
...@@ -71,10 +71,11 @@ based on NOMAD-coe's *python-common* module. ...@@ -71,10 +71,11 @@ based on NOMAD-coe's *python-common* module.
:members: :members:
""" """
from typing import Callable, IO, Union from typing import Callable, IO, Union, Dict
import magic import magic
import gzip import gzip
import bz2 import bz2
import lzma
import os.path import os.path
from nomad import files, config from nomad import files, config
...@@ -87,7 +88,8 @@ from nomad.parsing.artificial import TemplateParser, GenerateRandomParser, Chaos ...@@ -87,7 +88,8 @@ from nomad.parsing.artificial import TemplateParser, GenerateRandomParser, Chaos
_compressions = { _compressions = {
b'\x1f\x8b\x08': ('gz', gzip.open), b'\x1f\x8b\x08': ('gz', gzip.open),
b'\x42\x5a\x68': ('bz2', bz2.open) b'\x42\x5a\x68': ('bz2', bz2.open),
b'\xfd\x37\x7a': ('xz', lzma.open)
} }
...@@ -116,7 +118,7 @@ def match_parser(mainfile: str, upload_files: Union[str, files.StagingUploadFile ...@@ -116,7 +118,7 @@ def match_parser(mainfile: str, upload_files: Union[str, files.StagingUploadFile
with open(mainfile_path, 'rb') as f: with open(mainfile_path, 'rb') as f:
compression, open_compressed = _compressions.get(f.read(3), (None, open)) compression, open_compressed = _compressions.get(f.read(3), (None, open))
with open_compressed(mainfile_path, 'rb') as cf: with open_compressed(mainfile_path, 'rb') as cf: # type: ignore
buffer = cf.read(config.parser_matching_size) buffer = cf.read(config.parser_matching_size)
mime_type = magic.from_buffer(buffer, mime=True) mime_type = magic.from_buffer(buffer, mime=True)
...@@ -147,14 +149,14 @@ parsers = [ ...@@ -147,14 +149,14 @@ parsers = [
LegacyParser( LegacyParser(
name='parsers/vasp', code_name='VASP', name='parsers/vasp', code_name='VASP',
parser_class_name='vaspparser.VASPRunParserInterface', parser_class_name='vaspparser.VASPRunParserInterface',
mainfile_mime_re=r'(application/xml)|(text/.*)', mainfile_mime_re=r'(application/.*)|(text/.*)',
mainfile_contents_re=( mainfile_contents_re=(
r'^\s*<\?xml version="1\.0" encoding="ISO-8859-1"\?>\s*' r'^\s*<\?xml version="1\.0" encoding="ISO-8859-1"\?>\s*'
r'?\s*<modeling>' r'?\s*<modeling>'
r'?\s*<generator>' r'?\s*<generator>'
r'?\s*<i name="program" type="string">\s*vasp\s*</i>' r'?\s*<i name="program" type="string">\s*vasp\s*</i>'
r'?'), r'?'),
supported_compressions=['gz', 'bz2'] supported_compressions=['gz', 'bz2', 'xz']
), ),
VaspOutcarParser( VaspOutcarParser(
name='parsers/vasp-outcar', code_name='VASP', name='parsers/vasp-outcar', code_name='VASP',
......
...@@ -319,6 +319,13 @@ class SearchRequest: ...@@ -319,6 +319,13 @@ class SearchRequest:
value = quantity.elastic_value(value) value = quantity.elastic_value(value)
if quantity.elastic_search_type == 'terms':
if not isinstance(value, list):
value = [value]
self.q &= Q('terms', **{quantity.elastic_field: value})
return self
if isinstance(value, list): if isinstance(value, list):
values = value values = value
else: else:
......
apiVersion: v1 apiVersion: v1
appVersion: "0.6.0" appVersion: "0.7.0"
description: A Helm chart for Kubernetes that only runs nomad services and uses externally hosted databases. description: A Helm chart for Kubernetes that only runs nomad services and uses externally hosted databases.
name: nomad name: nomad
version: 0.6.0 version: 0.7.0
...@@ -39,6 +39,16 @@ data: ...@@ -39,6 +39,16 @@ data:
rewrite ^{{ .Values.proxy.external.path }}/gui/service-worker.js /nomad/service-worker.js break; rewrite ^{{ .Values.proxy.external.path }}/gui/service-worker.js /nomad/service-worker.js break;
} }
location {{ .Values.proxy.external.path }}/gui/meta.json {
add_header Last-Modified $date_gmt;
add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
if_modified_since off;
expires off;
etag off;
root /app/;
rewrite ^{{ .Values.proxy.external.path }}/gui/meta.json /nomad/meta.json break;
}
location {{ .Values.proxy.external.path }}/api/uploads { location {{ .Values.proxy.external.path }}/api/uploads {
client_max_body_size 35g; client_max_body_size 35g;
proxy_request_buffering off; proxy_request_buffering off;
......
...@@ -12,7 +12,7 @@ reqs = [str(ir.req) for ir in install_reqs if 'sphinxcontrib.httpdomain' not in ...@@ -12,7 +12,7 @@ reqs = [str(ir.req) for ir in install_reqs if 'sphinxcontrib.httpdomain' not in
setup( setup(
name='nomad', name='nomad',
version='0.6.0', version='0.7.0',
description='The nomad@FAIRDI infrastructure python package', description='The nomad@FAIRDI infrastructure python package',
py_modules=['nomad'], py_modules=['nomad'],
install_requires=reqs, install_requires=reqs,
......
...@@ -600,19 +600,19 @@ class TestRepo(): ...@@ -600,19 +600,19 @@ class TestRepo():
calc_with_metadata.update( calc_with_metadata.update(
calc_id='2', uploader=other_test_user.user_id, published=True, calc_id='2', uploader=other_test_user.user_id, published=True,
with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5), with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
external_id='external_id') external_id='external_2')
calc_with_metadata.update( calc_with_metadata.update(
atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz') atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True) search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
calc_with_metadata.update( calc_with_metadata.update(
calc_id='3', uploader=other_test_user.user_id, published=False, calc_id='3', uploader=other_test_user.user_id, published=False,
with_embargo=False, pid=3, external_id='external_id') with_embargo=False, pid=3, external_id='external_3')
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True) search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
calc_with_metadata.update( calc_with_metadata.update(
calc_id='4', uploader=other_test_user.user_id, published=True, calc_id='4', uploader=other_test_user.user_id, published=True,
with_embargo=True, pid=4, external_id='external_id') with_embargo=True, pid=4, external_id='external_4')
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True) search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
def assert_search(self, rv: Any, number_of_calcs: int) -> dict: def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
...@@ -711,30 +711,35 @@ class TestRepo(): ...@@ -711,30 +711,35 @@ class TestRepo():
rv = api.get('/repo/%s' % query_string) rv = api.get('/repo/%s' % query_string)
self.assert_search(rv, calcs) self.assert_search(rv, calcs)
@pytest.mark.parametrize('calcs, quantity, value', [ @pytest.mark.parametrize('calcs, quantity, value, user', [
(2, 'system', 'bulk'), (2, 'system', 'bulk', 'test_user'),
(0, 'system', 'atom'), (0, 'system', 'atom', 'test_user'),
(1, 'atoms', 'Br'), (1, 'atoms', 'Br', 'test_user'),
(1, 'atoms', 'Fe'), (1, 'atoms', 'Fe', 'test_user'),
(0, 'atoms', ['Fe', 'Br', 'A', 'B']), (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
(0, 'only_atoms', ['Br', 'Si']), (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
(1, 'only_atoms', ['Fe']), (1, 'only_atoms', ['Fe'], 'test_user'),
(1, 'only_atoms', ['Br', 'K', 'Si']), (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
(1, 'only_atoms', ['Br', 'Si', 'K']), (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
(1, 'comment', 'specific'), (1, 'comment', 'specific', 'test_user'),
(1, 'authors', 'Leonard Hofstadter'), (1, 'authors', 'Hofstadter, Leonard', 'test_user'),
(2, 'files', 'test/mainfile.txt'), (2, 'files', 'test/mainfile.txt', 'test_user'),
(2, 'paths', 'mainfile.txt'), (2, 'paths', 'mainfile.txt', 'test_user'),
(2, 'paths', 'test'), (2, 'paths', 'test', 'test_user'),
(2, 'quantities', ['wyckoff_letters_primitive', 'hall_number']), (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
(0, 'quantities', 'dos'), (0, 'quantities', 'dos', 'test_user'),
(1, 'external_id', 'external_id'), (2, 'external_id', 'external_2,external_3', 'other_test_user'),
(0, 'external_id', 'external') (1, 'external_id', 'external_2', 'test_user'),
(1, 'external_id', 'external_2,external_3', 'test_user'),
(0, 'external_id', 'external_x', 'test_user')
]) ])
def test_search_parameters(self, api, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value): def test_search_parameters(
self, api, example_elastic_calcs, no_warn, test_user_auth,
other_test_user_auth, calcs, quantity, value, user):
user_auth = test_user_auth if user == 'test_user' else other_test_user_auth
query_string = urlencode({quantity: value, 'statistics': True}, doseq=True) query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
rv = api.get('/repo/?%s' % query_string, headers=test_user_auth) rv = api.get('/repo/?%s' % query_string, headers=user_auth)
logger.debug('run search quantities test', query_string=query_string) logger.debug('run search quantities test', query_string=query_string)
data = self.assert_search(rv, calcs) data = self.assert_search(rv, calcs)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment