Commit f1f6d0e1 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merged latest v0.6.2 changes.

parents 81f52793 c8aed118
......@@ -198,8 +198,14 @@ class NavigationUnstyled extends React.Component {
}
componentDidMount() {
fetch(`${guiBase}/meta.json`)
.then((response) => response.json())
fetch(`${guiBase}/meta.json`, {
method: 'GET',
cache: 'no-cache',
headers: {
'Pragma': 'no-cache',
'Cache-Control': 'no-cache, no-store'
}
}).then((response) => response.json())
.then((meta) => {
if (meta.version !== packageJson.version) {
console.log('GUI API version mismatch')
......
......@@ -366,7 +366,7 @@ class RawFileQueryResource(Resource):
abort(400, message='bad parameter types')
search_request = search.SearchRequest()
add_query(search_request)
add_query(search_request, search_request_parser)
calcs = sorted([
(entry['upload_id'], entry['mainfile'])
......
......@@ -102,7 +102,7 @@ def add_common_parameters(request_parser):
for quantity in search.quantities.values():
request_parser.add_argument(
quantity.name, help=quantity.description,
action='append' if quantity.multi else None)
action=quantity.argparse_action if quantity.multi else None)
repo_request_parser = pagination_request_parser.copy()
......@@ -129,14 +129,16 @@ search_request_parser = api.parser()
add_common_parameters(search_request_parser)
def add_query(search_request: search.SearchRequest):
def add_query(search_request: search.SearchRequest, parser=repo_request_parser):
"""
Help that adds query relevant request parameters to the given SearchRequest.
"""
args = {key: value for key, value in parser.parse_args().items() if value is not None}
# owner
try:
search_request.owner(
request.args.get('owner', 'all'),
args.get('owner', 'all'),
g.user.user_id if g.user is not None else None)
except ValueError as e:
abort(401, getattr(e, 'message', 'Invalid owner parameter'))
......@@ -144,8 +146,8 @@ def add_query(search_request: search.SearchRequest):
abort(400, getattr(e, 'message', 'Invalid owner parameter'))
# time range
from_time_str = request.args.get('from_time', None)
until_time_str = request.args.get('until_time', None)
from_time_str = args.get('from_time', None)
until_time_str = args.get('until_time', None)
try:
from_time = rfc3339DateTime.parse(from_time_str) if from_time_str is not None else None
......@@ -156,7 +158,7 @@ def add_query(search_request: search.SearchRequest):
# optimade
try:
optimade = request.args.get('optimade', None)
optimade = args.get('optimade', None)
if optimade is not None:
q = filterparser.parse_filter(optimade)
search_request.query(q)
......@@ -165,8 +167,7 @@ def add_query(search_request: search.SearchRequest):
# search parameter
search_request.search_parameters(**{
key: request.args.getlist(key) if search.quantities[key] else request.args.get(key)
for key in request.args.keys()
key: value for key, value in args.items()
if key not in ['optimade'] and key in search.quantities})
......@@ -210,7 +211,7 @@ class RepoCalcsResource(Resource):
"""
search_request = search.SearchRequest()
add_query(search_request)
add_query(search_request, repo_request_parser)
try:
scroll = bool(request.args.get('scroll', False))
......@@ -325,7 +326,7 @@ class RepoQuantityResource(Resource):
"""
search_request = search.SearchRequest()
add_query(search_request)
add_query(search_request, repo_quantity_search_request_parser)
try:
after = request.args.get('after', None)
......
......@@ -222,6 +222,7 @@ class UploadListResource(Resource):
pagination=dict(total=total, page=page, per_page=per_page),
results=results), 200
@api.doc(security=list(api.authorizations.keys())) # weird bug, this should not be necessary
@api.doc('upload')
@api.expect(upload_metadata_parser)
@api.response(400, 'To many uploads')
......
......@@ -187,7 +187,7 @@ client = NomadConfig(
url='http://localhost:8000/fairdi/nomad/latest/api'
)
version = '0.6.0'
version = '0.7.0'
commit = gitinfo.commit
release = 'devel'
domain = 'DFT'
......
......@@ -260,6 +260,7 @@ class DomainQuantity:
elastic_field: An optional elasticsearch key. Default is the name of the quantity.
elastic_value: A collable that takes a :class:`CalcWithMetadata` as input and produces the
value for the elastic search index.
argparse_action: Action to use on argparse, either append or split for multi values. Append is default.
"""
def __init__(
......@@ -268,7 +269,8 @@ class DomainQuantity:
zero_aggs: bool = True, metadata_field: str = None,
elastic_mapping: type = None,
elastic_search_type: str = 'term', elastic_field: str = None,
elastic_value: Callable[[Any], Any] = None):
elastic_value: Callable[[Any], Any] = None,
argparse_action: str = 'append'):
self._name: str = None
self.description = description
......@@ -281,6 +283,7 @@ class DomainQuantity:
self.elastic_search_type = elastic_search_type
self.metadata_field = metadata_field
self.elastic_field = elastic_field
self.argparse_action = argparse_action
self.elastic_value = elastic_value
if self.elastic_value is None:
......@@ -353,7 +356,9 @@ class Domain:
pid=DomainQuantity(description='Search for the pid.'),
raw_id=DomainQuantity(description='Search for the raw_id.'),
mainfile=DomainQuantity(description='Search for the mainfile.'),
external_id=DomainQuantity(description='External user provided id. Does not have to be unique necessarily.'),
external_id=DomainQuantity(
description='External user provided id. Does not have to be unique necessarily.',
multi=True, argparse_action='split', elastic_search_type='terms'),
dataset=DomainQuantity(
elastic_field='datasets.name', multi=True, elastic_search_type='match',
description='Search for a particular dataset by name.'),
......
......@@ -71,10 +71,11 @@ based on NOMAD-coe's *python-common* module.
:members:
"""
from typing import Callable, IO, Union
from typing import Callable, IO, Union, Dict
import magic
import gzip
import bz2
import lzma
import os.path
from nomad import files, config
......@@ -87,7 +88,8 @@ from nomad.parsing.artificial import TemplateParser, GenerateRandomParser, Chaos
_compressions = {
b'\x1f\x8b\x08': ('gz', gzip.open),
b'\x42\x5a\x68': ('bz2', bz2.open)
b'\x42\x5a\x68': ('bz2', bz2.open),
b'\xfd\x37\x7a': ('xz', lzma.open)
}
......@@ -116,7 +118,7 @@ def match_parser(mainfile: str, upload_files: Union[str, files.StagingUploadFile
with open(mainfile_path, 'rb') as f:
compression, open_compressed = _compressions.get(f.read(3), (None, open))
with open_compressed(mainfile_path, 'rb') as cf:
with open_compressed(mainfile_path, 'rb') as cf: # type: ignore
buffer = cf.read(config.parser_matching_size)
mime_type = magic.from_buffer(buffer, mime=True)
......@@ -147,14 +149,14 @@ parsers = [
LegacyParser(
name='parsers/vasp', code_name='VASP',
parser_class_name='vaspparser.VASPRunParserInterface',
mainfile_mime_re=r'(application/xml)|(text/.*)',
mainfile_mime_re=r'(application/.*)|(text/.*)',
mainfile_contents_re=(
r'^\s*<\?xml version="1\.0" encoding="ISO-8859-1"\?>\s*'
r'?\s*<modeling>'
r'?\s*<generator>'
r'?\s*<i name="program" type="string">\s*vasp\s*</i>'
r'?'),
supported_compressions=['gz', 'bz2']
supported_compressions=['gz', 'bz2', 'xz']
),
VaspOutcarParser(
name='parsers/vasp-outcar', code_name='VASP',
......
......@@ -319,6 +319,13 @@ class SearchRequest:
value = quantity.elastic_value(value)
if quantity.elastic_search_type == 'terms':
if not isinstance(value, list):
value = [value]
self.q &= Q('terms', **{quantity.elastic_field: value})
return self
if isinstance(value, list):
values = value
else:
......
apiVersion: v1
appVersion: "0.6.0"
appVersion: "0.7.0"
description: A Helm chart for Kubernetes that only runs nomad services and uses externally hosted databases.
name: nomad
version: 0.6.0
version: 0.7.0
......@@ -39,6 +39,16 @@ data:
rewrite ^{{ .Values.proxy.external.path }}/gui/service-worker.js /nomad/service-worker.js break;
}
location {{ .Values.proxy.external.path }}/gui/meta.json {
add_header Last-Modified $date_gmt;
add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
if_modified_since off;
expires off;
etag off;
root /app/;
rewrite ^{{ .Values.proxy.external.path }}/gui/meta.json /nomad/meta.json break;
}
location {{ .Values.proxy.external.path }}/api/uploads {
client_max_body_size 35g;
proxy_request_buffering off;
......
......@@ -12,7 +12,7 @@ reqs = [str(ir.req) for ir in install_reqs if 'sphinxcontrib.httpdomain' not in
setup(
name='nomad',
version='0.6.0',
version='0.7.0',
description='The nomad@FAIRDI infrastructure python package',
py_modules=['nomad'],
install_requires=reqs,
......
......@@ -600,19 +600,19 @@ class TestRepo():
calc_with_metadata.update(
calc_id='2', uploader=other_test_user.user_id, published=True,
with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
external_id='external_id')
external_id='external_2')
calc_with_metadata.update(
atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
calc_with_metadata.update(
calc_id='3', uploader=other_test_user.user_id, published=False,
with_embargo=False, pid=3, external_id='external_id')
with_embargo=False, pid=3, external_id='external_3')
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
calc_with_metadata.update(
calc_id='4', uploader=other_test_user.user_id, published=True,
with_embargo=True, pid=4, external_id='external_id')
with_embargo=True, pid=4, external_id='external_4')
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
......@@ -711,30 +711,35 @@ class TestRepo():
rv = api.get('/repo/%s' % query_string)
self.assert_search(rv, calcs)
@pytest.mark.parametrize('calcs, quantity, value', [
(2, 'system', 'bulk'),
(0, 'system', 'atom'),
(1, 'atoms', 'Br'),
(1, 'atoms', 'Fe'),
(0, 'atoms', ['Fe', 'Br', 'A', 'B']),
(0, 'only_atoms', ['Br', 'Si']),
(1, 'only_atoms', ['Fe']),
(1, 'only_atoms', ['Br', 'K', 'Si']),
(1, 'only_atoms', ['Br', 'Si', 'K']),
(1, 'comment', 'specific'),
(1, 'authors', 'Leonard Hofstadter'),
(2, 'files', 'test/mainfile.txt'),
(2, 'paths', 'mainfile.txt'),
(2, 'paths', 'test'),
(2, 'quantities', ['wyckoff_letters_primitive', 'hall_number']),
(0, 'quantities', 'dos'),
(1, 'external_id', 'external_id'),
(0, 'external_id', 'external')
@pytest.mark.parametrize('calcs, quantity, value, user', [
(2, 'system', 'bulk', 'test_user'),
(0, 'system', 'atom', 'test_user'),
(1, 'atoms', 'Br', 'test_user'),
(1, 'atoms', 'Fe', 'test_user'),
(0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
(0, 'only_atoms', ['Br', 'Si'], 'test_user'),
(1, 'only_atoms', ['Fe'], 'test_user'),
(1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
(1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
(1, 'comment', 'specific', 'test_user'),
(1, 'authors', 'Hofstadter, Leonard', 'test_user'),
(2, 'files', 'test/mainfile.txt', 'test_user'),
(2, 'paths', 'mainfile.txt', 'test_user'),
(2, 'paths', 'test', 'test_user'),
(2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
(0, 'quantities', 'dos', 'test_user'),
(2, 'external_id', 'external_2,external_3', 'other_test_user'),
(1, 'external_id', 'external_2', 'test_user'),
(1, 'external_id', 'external_2,external_3', 'test_user'),
(0, 'external_id', 'external_x', 'test_user')
])
def test_search_parameters(self, api, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
def test_search_parameters(
self, api, example_elastic_calcs, no_warn, test_user_auth,
other_test_user_auth, calcs, quantity, value, user):
user_auth = test_user_auth if user == 'test_user' else other_test_user_auth
query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
rv = api.get('/repo/?%s' % query_string, headers=test_user_auth)
rv = api.get('/repo/?%s' % query_string, headers=user_auth)
logger.debug('run search quantities test', query_string=query_string)
data = self.assert_search(rv, calcs)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment