diff --git a/nomad/app/api/common.py b/nomad/app/api/common.py
index 7174d0c7ca78067fa0d03650a811f9b7ee4646a1..a75844700e4c1baf11a2d784c62ce0c33f24e178 100644
--- a/nomad/app/api/common.py
+++ b/nomad/app/api/common.py
@@ -98,7 +98,7 @@ query_model_fields = {
query_model_fields.update(**{
'owner': fields.String(description='The group the calculations belong to.', allow_null=True, skip_none=True),
'domain': fields.String(description='Specify the domain to search in: %s, default is ``%s``' % (
- ', '.join(['``%s``' % domain for domain in datamodel.domains]), config.default_domain)),
+ ', '.join(['``%s``' % domain for domain in datamodel.domains]), config.meta.default_domain)),
'from_time': fields.Raw(description='The minimum entry time.', allow_null=True, skip_none=True),
'until_time': fields.Raw(description='The maximum entry time.', allow_null=True, skip_none=True)
})
@@ -138,7 +138,7 @@ def add_search_parameters(request_parser):
'domain', type=str,
help='Specify the domain to search in: %s, default is ``%s``' % (
', '.join(['``%s``' % domain for domain in datamodel.domains]),
- config.default_domain))
+ config.meta.default_domain))
request_parser.add_argument(
'owner', type=str,
help='Specify which calcs to return: ``visible``, ``public``, ``all``, ``user``, ``staging``, default is ``visible``')
@@ -338,7 +338,7 @@ def query_api_clientlib(**kwargs):
kwargs = {
key: normalize_value(key, value) for key, value in kwargs.items()
- if key in search.search_quantities and (key != 'domain' or value != config.default_domain)
+ if key in search.search_quantities and (key != 'domain' or value != config.meta.default_domain)
}
out = io.StringIO()
diff --git a/nomad/app/api/info.py b/nomad/app/api/info.py
index 73521c26b0d20e0db43fbca53d995018c7d45e60..ce67893f0063c05f5a88fcc1cbdaf7510027699a 100644
--- a/nomad/app/api/info.py
+++ b/nomad/app/api/info.py
@@ -131,8 +131,8 @@ class InfoResource(Resource):
for s in search.search_quantities.values()
if 'optimade' not in s.qualified_name
},
- 'version': config.version,
- 'release': config.release,
+ 'version': config.meta.version,
+ 'release': config.meta.release,
'git': {
'ref': gitinfo.ref,
'version': gitinfo.version,
diff --git a/nomad/app/optimade/api.py b/nomad/app/optimade/api.py
index 88819e0f9f0bbb99f3502bd41db1a5a18e62ef02..c58555bc35084571a582cdc6a44513f26938b204 100644
--- a/nomad/app/optimade/api.py
+++ b/nomad/app/optimade/api.py
@@ -25,18 +25,20 @@ base_url = 'http://%s/%s/optimade' % (
config.services.api_base_path.strip('/'))
-def url(endpoint: str = None, version='v0', **kwargs):
+def url(endpoint: str = None, version='v0', prefix=None, **kwargs):
''' Returns the full optimade api url (for a given endpoint) including query parameters. '''
- if endpoint is None:
- if version is not None:
- url = '%s/%s' % (base_url, version)
- else:
- url = base_url
+ if endpoint is not None:
+ url = '/' + endpoint
else:
- if version is not None:
- url = '%s/%s/%s' % (base_url, version, endpoint)
- else:
- url = '%s/%s' % (base_url, endpoint)
+ url = ''
+
+ if version is not None:
+ url = '/' + version + url
+
+ if prefix is not None:
+ url = '/' + prefix + url
+
+ url = base_url + url
if len(kwargs) > 0:
return '%s?%s' % (url, urllib.parse.urlencode(kwargs))
diff --git a/nomad/app/optimade/endpoints.py b/nomad/app/optimade/endpoints.py
index 93bdce94d9c5f2c8aa062d4cf1236bc75b079ba3..b2bb8521669cec29126f425305af38828224a7a0 100644
--- a/nomad/app/optimade/endpoints.py
+++ b/nomad/app/optimade/endpoints.py
@@ -17,14 +17,14 @@ from flask_restplus import Resource, abort
from flask import request
from elasticsearch_dsl import Q
-from nomad import search, files, datamodel
+from nomad import search, files, datamodel, config
from nomad.datamodel import OptimadeEntry
from .api import api, url, base_request_args
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
Links as LinksModel, CalculationDataObject, single_entry_endpoint_parser, base_endpoint_parser, \
json_api_info_response_model, json_api_list_response_model, ReferenceObject, StructureObject, \
- ToplevelLinks, LinkObject, json_api_links_response_model, json_api_references_response_model, \
+ ToplevelLinks, json_api_references_response_model, \
json_api_structure_response_model, json_api_structures_response_model
from .filterparser import parse_filter, FilterException
@@ -274,43 +274,31 @@ class References(Resource):
class Links(Resource):
@api.doc('links')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
- @api.response(422, 'Validation error')
- @api.expect(entry_listing_endpoint_parser, validate=True)
- @api.marshal_with(json_api_links_response_model, skip_none=True, code=200)
+ @api.expect(base_endpoint_parser, validate=True)
+ @api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
def get(self):
- '''Retrive the links that corresponding to the structures that match the given Optimade filter expression'''
- try:
- filter = request.args.get('filter', None)
- page_limit = int(request.args.get('page_limit', 10))
- page_number = int(request.args.get('page_number', 1))
- sort = request.args.get('sort', 'chemical_formula_reduced'),
-
- except Exception:
- abort(400, message='bad parameter types') # TODO Specific json API error handling
+ ''' Returns information relating to the API implementation- '''
+ base_request_args()
- result = execute_search(
- filter=filter, page_limit=page_limit, page_number=page_number, sort=sort)
- available = result['pagination']['total']
- results = to_calc_with_metadata(result['results'])
- assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
+ result = [
+ {
+ "type": "parent",
+ "id": "index",
+ "attributes": {
+ "name": config.meta.name,
+ "description": config.meta.description,
+ "base_url": {
+ "href": url(version=None, prefix='index'),
+ },
+ "homepage": config.meta.homepage
+ }
+ }
+ ]
return dict(
- meta=Meta(
- query=request.url,
- returned=len(results),
- available=available,
- last_id=results[-1].calc_id if available > 0 else None),
- links=ToplevelLinks(
- 'structures',
- available=available,
- page_number=page_number,
- page_limit=page_limit,
- sort=sort, filter=filter
- ),
- # TODO Links are about links to other optimade databases, e.g. OQMD, MP, AFLOW.
- # It is not about links within NOMAD, like LinkObject suggests.
- data=[LinkObject(d, page_number=page_number, sort=sort, filter=filter) for d in results]
- )
+ meta=Meta(query=request.url, returned=1),
+ data=result
+ ), 200
@ns.route('/structures')
diff --git a/nomad/app/optimade/index.py b/nomad/app/optimade/index.py
index 19febfbc7a41b45c9e9406af0a4e36a5748060bf..0d43260bbdbceb402ba2a1d3f5bee645f78fa756 100644
--- a/nomad/app/optimade/index.py
+++ b/nomad/app/optimade/index.py
@@ -15,6 +15,8 @@
from flask_restplus import Resource
from flask import request
+from nomad import config
+
from .api import api, url, base_request_args
from .models import json_api_single_response_model, base_endpoint_parser, json_api_single_response_model, Meta, json_api_list_response_model
@@ -57,7 +59,7 @@ class Info(Resource):
@ns.route('/links')
class Links(Resource):
- @api.doc('index_info')
+ @api.doc('index_links')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
@@ -66,31 +68,21 @@ class Links(Resource):
base_request_args()
result = [
- {
- "type": "parent",
- "id": "index",
- "attributes": {
- "name": "NOMAD OPTiMaDe index",
- "description": "Index for NOMAD's OPTiMaDe implemenations",
- "base_url": url(version=None),
- "homepage": "http://nomad-coe.eu"
- }
- },
{
"type": "child",
"id": "v0",
"attributes": {
- "name": "NOMAD OPTiMaDe v0",
- "description": "Novel Materials Discovery OPTiMaDe implementations v0",
+ "name": config.meta.name,
+ "description": config.meta.description,
"base_url": {
"href": url(),
},
- "homepage": "http://nomad-coe.eu"
+ "homepage": config.meta.homepage
}
}
]
return dict(
- meta=Meta(query=request.url, returned=2),
+ meta=Meta(query=request.url, returned=1),
data=result
), 200
diff --git a/nomad/app/optimade/models.py b/nomad/app/optimade/models.py
index dcbadf79e112bf7420eb7898055d424535d4044c..426b3c577c52562e3bb11e23ce8d8304d76dfcc3 100644
--- a/nomad/app/optimade/models.py
+++ b/nomad/app/optimade/models.py
@@ -128,7 +128,7 @@ class Meta():
self.last_id = last_id
self.implementation = dict(
name='nomad@fairdi',
- version=config.version,
+ version=config.meta.version,
source_url='https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR',
maintainer=dict(email='markus.scheidgen@physik.hu-berlin.de'))
diff --git a/nomad/cli/admin/run.py b/nomad/cli/admin/run.py
index f775ee99c0038458373edfb5fa312d13c1c196e5..d2f78e91db57ecc2608bfead9d645eda0360b518 100644
--- a/nomad/cli/admin/run.py
+++ b/nomad/cli/admin/run.py
@@ -39,7 +39,7 @@ def app(debug: bool, with_chaos: int):
def run_app(**kwargs):
- config.service = 'app'
+ config.meta.service = 'app'
from nomad import infrastructure
from nomad.app.__main__ import run_dev_server
infrastructure.setup()
@@ -47,7 +47,7 @@ def run_app(**kwargs):
def run_worker():
- config.service = 'worker'
+ config.meta.service = 'worker'
from nomad import processing
processing.app.worker_main(['worker', '--loglevel=INFO', '-Q', 'celery,uploads,calcs'])
diff --git a/nomad/cli/admin/uploads.py b/nomad/cli/admin/uploads.py
index 60fbe23ef432fd3442be854665ed93f4f8f1acb8..d974f07fb292f9d3db06045b4b8bf02b226b98f0 100644
--- a/nomad/cli/admin/uploads.py
+++ b/nomad/cli/admin/uploads.py
@@ -60,7 +60,7 @@ def uploads(
if outdated:
uploads = proc.Calc._get_collection().distinct(
'upload_id',
- {'metadata.nomad_version': {'$ne': config.version}})
+ {'metadata.nomad_version': {'$ne': config.meta.version}})
query |= mongoengine.Q(upload_id__in=uploads)
if code is not None and len(code) > 0:
diff --git a/nomad/cli/cli.py b/nomad/cli/cli.py
index a18d9d4768b0374885aaeccb5826c468b286c3d5..6137fdd115d1e3fc46f5524e4eaade7d6b67dbc2 100644
--- a/nomad/cli/cli.py
+++ b/nomad/cli/cli.py
@@ -97,7 +97,7 @@ class POPO(dict):
@click.option('--config', help='the config file to use')
@click.pass_context
def cli(ctx, verbose: bool, debug: bool, config: str):
- nomad_config.service = os.environ.get('NOMAD_SERVICE', 'cli')
+ nomad_config.meta.service = os.environ.get('NOMAD_SERVICE', 'cli')
if config is not None:
nomad_config.load_config(config_file=config)
diff --git a/nomad/config.py b/nomad/config.py
index 019db0280882b42b7c8e0660e7fa6d6c89ead9b0..535d26b744a02cde7b01f085903ca296b70d680c 100644
--- a/nomad/config.py
+++ b/nomad/config.py
@@ -261,11 +261,17 @@ datacite = NomadConfig(
password='*'
)
-version = '0.8.1'
-commit = gitinfo.commit
-release = 'devel'
-default_domain = 'dft'
-service = 'unknown nomad service'
+meta = NomadConfig(
+ version='0.8.1',
+ commit=gitinfo.commit,
+ release='devel',
+ default_domain='dft',
+ service='unknown nomad service',
+ name='novel materials discovery (NOMAD)',
+ description='A FAIR data sharing platform for materials science data',
+ homepage='https://nomad-coe.eu'
+)
+
auxfile_cutoff = 100
parser_matching_size = 9128
console_log_level = logging.WARNING
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index b9782abff69c0b6d754f6b4ad3d65aa245b840af..69a54cdf7bdca39e7404727047073eff53cc5561 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -150,8 +150,8 @@ class Calc(Proc):
entry_metadata.upload_id = self.upload_id
entry_metadata.calc_id = self.calc_id
entry_metadata.mainfile = self.mainfile
- entry_metadata.nomad_version = config.version
- entry_metadata.nomad_commit = config.commit
+ entry_metadata.nomad_version = config.meta.version
+ entry_metadata.nomad_commit = config.meta.commit
entry_metadata.uploader = self.upload.user_id
entry_metadata.upload_time = self.upload.upload_time
entry_metadata.upload_name = self.upload.name
@@ -280,8 +280,8 @@ class Calc(Proc):
try:
self._entry_metadata = self.user_metadata()
self._entry_metadata.calc_hash = self.upload_files.calc_hash(self.mainfile)
- self._entry_metadata.nomad_version = config.version
- self._entry_metadata.nomad_commit = config.commit
+ self._entry_metadata.nomad_version = config.meta.version
+ self._entry_metadata.nomad_commit = config.meta.commit
self._entry_metadata.last_processing = datetime.utcnow()
self._entry_metadata.files = self.upload_files.calc_files(self.mainfile)
@@ -1151,7 +1151,7 @@ class Upload(Proc):
''' All successfully processed and outdated calculations. '''
return Calc.objects(
upload_id=self.upload_id, tasks_status=SUCCESS,
- metadata__nomad_version__ne=config.version)
+ metadata__nomad_version__ne=config.meta.version)
@property
def calcs(self):
diff --git a/nomad/search.py b/nomad/search.py
index 275ac17947ebd448c0ba36f4a75b265719261a77..06cec0bbcc6f2e60f857b375a17577f06bafa5e9 100644
--- a/nomad/search.py
+++ b/nomad/search.py
@@ -130,7 +130,7 @@ class SearchRequest:
There is also scrolling for quantities to go through all quantity values. There is no
paging for aggregations.
'''
- def __init__(self, domain: str = config.default_domain, query=None):
+ def __init__(self, domain: str = config.meta.default_domain, query=None):
self._domain = domain
self._query = query
self._search = Search(index=config.elastic.index_name)
diff --git a/nomad/utils/structlogging.py b/nomad/utils/structlogging.py
index a513a43e1ce36d981d1cb1875d68e645e57cc1f8..b5e1086628efff2207e3c5f0fed820918dc92e5c 100644
--- a/nomad/utils/structlogging.py
+++ b/nomad/utils/structlogging.py
@@ -144,8 +144,8 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase):
'logger_name': record.name,
# Nomad specific
- 'nomad.service': config.service,
- 'nomad.release': config.release
+ 'nomad.service': config.meta.service,
+ 'nomad.release': config.meta.release
}
if record.name.startswith('nomad'):
@@ -238,7 +238,7 @@ def add_logstash_handler(logger):
logstash_handler = LogstashHandler(
config.logstash.host,
config.logstash.tcp_port, version=1)
- logstash_handler.formatter = LogstashFormatter(tags=['nomad', config.release])
+ logstash_handler.formatter = LogstashFormatter(tags=['nomad', config.meta.release])
logstash_handler.setLevel(config.logstash.level)
logger.addHandler(logstash_handler)
diff --git a/setup.py b/setup.py
index d285fa289530b4e4dd1cd53dd843575ca832e6e4..073ff12a15c61222c6a22964bfbf59ae72e3bb3d 100644
--- a/setup.py
+++ b/setup.py
@@ -233,7 +233,7 @@ def setup_kwargs():
return dict(
name='nomad-lab',
- version=config.version,
+ version=config.meta.version,
description='The NOvel MAterials Discovery (NOMAD) Python package',
package_dir={'': './'},
packages=['nomad.%s' % pkg for pkg in find_packages('./nomad')] + ['nomad'],
diff --git a/tests/app/test_api.py b/tests/app/test_api.py
index 1d2dfd0e09b02bd1effcddceb70bfcf3ffa26c56..f77d375160220507667a7abc6b66622a5bc2d41f 100644
--- a/tests/app/test_api.py
+++ b/tests/app/test_api.py
@@ -475,8 +475,8 @@ class TestUploads:
self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, {})
def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
- monkeypatch.setattr('nomad.config.version', 're_process_test_version')
- monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')
+ monkeypatch.setattr('nomad.config.meta.version', 're_process_test_version')
+ monkeypatch.setattr('nomad.config.meta.commit', 're_process_test_commit')
upload_id = published.upload_id
rv = api.post(
diff --git a/tests/app/test_optimade.py b/tests/app/test_optimade.py
index 26c91689401f0459fa2d09bc55b5b41907bdd9ca..503f690d9001b81d933292347c471ab8a33dac31 100644
--- a/tests/app/test_optimade.py
+++ b/tests/app/test_optimade.py
@@ -41,7 +41,7 @@ def test_index(index_api):
rv = index_api.get('/links')
assert rv.status_code == 200
data = json.loads(rv.data)
- assert data['data'][1]['attributes']['base_url']['href'].endswith('v0')
+ assert data['data'][0]['attributes']['base_url']['href'].endswith('optimade/v0')
def test_get_entry(published: Upload):
@@ -247,13 +247,7 @@ def test_links_endpoint(api, example_structures):
rv = api.get('/links')
assert rv.status_code == 200
data = json.loads(rv.data)
- assert 'data' in data
- assert len(data['data']) == 4
- for d in data['data']:
- for key in ['id', 'type', 'attributes']:
- assert d.get(key) is not None
- for key in ['name', 'description', 'base_url', 'homepage']:
- assert key in d['attributes']
+ assert data['data'][0]['attributes']['base_url']['href'].endswith('optimade/index')
def test_structures_endpoint(api, example_structures):
diff --git a/tests/conftest.py b/tests/conftest.py
index 1e4796f18ccece6e55540da875158777fb0322d5..3ac1a6a6425ff26d7fa7c15ff91120b8501093d4 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -29,6 +29,7 @@ import elasticsearch.exceptions
from typing import List
import json
import logging
+import warnings
from nomad import config, infrastructure, parsing, processing, app, utils
from nomad.utils import structlogging
@@ -43,6 +44,8 @@ from tests.bravado_flask import FlaskTestHttpClient
test_log_level = logging.CRITICAL
example_files = [empty_file, example_file]
+warnings.simplefilter("ignore")
+
structlogging.ConsoleFormatter.short_format = True
setattr(logging, 'Formatter', structlogging.ConsoleFormatter)
@@ -626,9 +629,9 @@ def published_wo_user_metadata(non_empty_processed: processing.Upload) -> proces
@pytest.fixture
def reset_config():
''' Fixture that resets configuration. '''
- service = config.service
+ service = config.meta.service
yield None
- config.service = service
+ config.meta.service = service
utils.set_console_log_level(test_log_level)
diff --git a/tests/parser_measurement.py b/tests/parser_measurement.py
index 3a013388ebe14272c324026312d6055dc8c8f5b9..e0cec2adac49c8dcca22cb1125ec0d590effa8c8 100644
--- a/tests/parser_measurement.py
+++ b/tests/parser_measurement.py
@@ -26,8 +26,8 @@ if __name__ == '__main__':
logger.error('parsing was not successful', status=backend.status)
backend.openNonOverlappingSection('section_entry_info')
- backend.addValue('upload_id', config.services.unavailable_value)
- backend.addValue('calc_id', config.services.unavailable_value)
+ backend.addValue('upload_id', config.meta.services.unavailable_value)
+ backend.addValue('calc_id', config.meta.services.unavailable_value)
backend.addValue('calc_hash', "no hash")
backend.addValue('mainfile', mainfile_path)
backend.addValue('parser_name', 'parsers/vasp')
diff --git a/tests/processing/test_data.py b/tests/processing/test_data.py
index 8f0ed36131b124735da99070f98a3c4ec198c070..af9ee6c9dd122a4b44bbad037d520478c83ed7dd 100644
--- a/tests/processing/test_data.py
+++ b/tests/processing/test_data.py
@@ -260,8 +260,8 @@ def test_re_processing(published: Upload, internal_example_user_metadata, monkey
raw_files, published.upload_files.join_file('raw-restricted.plain.zip').os_path)
# reprocess
- monkeypatch.setattr('nomad.config.version', 're_process_test_version')
- monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')
+ monkeypatch.setattr('nomad.config.meta.version', 're_process_test_version')
+ monkeypatch.setattr('nomad.config.meta.commit', 're_process_test_commit')
published.reset()
published.re_process_upload()
try:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 2702c7c5da4d1af84c5c175c7d7fde2ab12a6c47..f51b9f89b9a2e3492515bf637eb33d3c3d98a1e1 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -233,7 +233,7 @@ class TestAdminUploads:
assert search.SearchRequest().search_parameters(comment='specific').execute()['total'] == 1
def test_re_process(self, published, monkeypatch):
- monkeypatch.setattr('nomad.config.version', 'test_version')
+ monkeypatch.setattr('nomad.config.meta.version', 'test_version')
upload_id = published.upload_id
calc = Calc.objects(upload_id=upload_id).first()
assert calc.metadata['nomad_version'] != 'test_version'