All tests working after domain refactor.

nomad/app/api/info.py

@@ -69,25 +69,30 @@ class InfoResource(Resource):
     @api.marshal_with(info_model, skip_none=True, code=200, description='Info send')
     def get(self):
         """ Return information about the nomad backend and its configuration. """
+        codes = [
+            parser.code_name
+            for parser in parsing.parser_dict.values()
+            if isinstance(parser, parsing.MatchingParser) and parser.domain == 'dft']
+
         return {
             'parsers': [
                 key[key.index('/') + 1:]
                 for key in parsing.parser_dict.keys()],
-            'codes': sorted(set([
-                parser.code_name
-                for parser in parsing.parser_dict.values()
-                if isinstance(parser, parsing.MatchingParser) and parser.domain == datamodel.Domain.instance.name]), key=lambda x: x.lower()),
+            'codes': sorted(set(codes), key=lambda x: x.lower()),
             'normalizers': [normalizer.__name__ for normalizer in normalizing.normalizers],
-            'domain': {
-                'name': datamodel.Domain.instance.name,
-                'quantities': [quantity for quantity in datamodel.Domain.instance.quantities.values()],
-                'metrics_names': datamodel.Domain.instance.metrics_names,
-                'aggregations_names': datamodel.Domain.instance.aggregations_names,
-                'metainfo': {
-                    'all_package': datamodel.Domain.instance.metainfo_all_package,
-                    'root_sections': datamodel.Domain.instance.root_sections
+            'domains': [
+                {
+                    'name': domain.name,
+                    'quantities': [quantity for quantity in domain.quantities.values()],
+                    'metrics_names': domain.metrics_names,
+                    'aggregations_names': domain.aggregations_names,
+                    'metainfo': {
+                        'all_package': domain.metainfo_all_package,
+                        'root_sections': domain.root_sections
+                    }
                 }
-            },
+                for domain in datamodel.Domain.instances.values()
+            ],
             'version': config.version,
             'release': config.release,
             'git': {

nomad/app/api/repo.py

@@ -148,7 +148,8 @@ class RepoCalcsResource(Resource):
         The search will return aggregations on a predefined set of quantities. Aggregations
         will tell you what quantity values exist and how many entries match those values.
 
-        Ordering is determined by ``order_by`` and ``order`` parameters.
+        Ordering is determined by ``order_by`` and ``order`` parameters. Default is
+        ``upload_time`` in decending order.
         """
 
         try:
@@ -162,7 +163,7 @@ class RepoCalcsResource(Resource):
             page = args.get('page', 1)
             per_page = args.get('per_page', 10 if not scroll else 1000)
             order = args.get('order', -1)
-            order_by = args.get('order_by', 'formula')
+            order_by = args.get('order_by', 'upload_time')
 
             date_histogram = args.get('date_histogram', False)
             metrics: List[str] = request.args.getlist('metrics')
@@ -594,9 +595,8 @@ class RepoQuantityResource(Resource):
         except AssertionError:
             abort(400, message='invalid size')
 
-        search_request.quantity(quantity, size=size, after=after)
-
         try:
+            search_request.quantity(quantity, size=size, after=after)
             results = search_request.execute()
             quantities = results.pop('quantities')
             results['quantity'] = quantities[quantity]

nomad/app/optimade/endpoints.py

@@ -43,7 +43,7 @@ def base_request_args():
 def base_search_request():
     """ Creates a search request for all public and optimade enabled data. """
     return search.SearchRequest().owner('all', None).query(
-        Q('exists', field='optimade.nelements'))  # TODO use the elastic annotations when done
+        Q('exists', field='dft.optimade.nelements'))  # TODO use the elastic annotations when done
 
 
 @ns.route('/calculations')

nomad/app/optimade/filterparser.py

@@ -26,7 +26,7 @@ class FilterException(Exception):
 
 quantities: Dict[str, Quantity] = {
     q.name: Quantity(
-        q.name, es_field='optimade.%s' % q.name,
+        q.name, es_field='dft.optimade.%s' % q.name,
         elastic_mapping_type=q.m_annotations['elastic']['type'])
 
     for q in OptimadeEntry.m_def.all_quantities.values()
@@ -34,7 +34,7 @@ quantities: Dict[str, Quantity] = {
 
 quantities['elements'].length_quantity = quantities['nelements']
 quantities['dimension_types'].length_quantity = quantities['dimension_types']
-quantities['elements'].has_only_quantity = Quantity(name='only_atoms')
+quantities['elements'].has_only_quantity = Quantity(name='dft.only_atoms')
 quantities['elements'].nested_quantity = quantities['elements_ratios']
 quantities['elements_ratios'].nested_quantity = quantities['elements_ratios']
 

nomad/cli/admin/uploads.py

@@ -50,7 +50,7 @@ def uploads(ctx, user: str, staging: bool, processing: bool, outdated: bool, cod
         query &= Q(upload_id__in=uploads)
 
     if code is not None and len(code) > 0:
-        code_queries = [es.Q('match', code_name=code_name) for code_name in code]
+        code_queries = [es.Q('match', **{'dft.code_name': code_name}) for code_name in code]
         code_query = es.Q('bool', should=code_queries, minimum_should_match=1)
 
         code_search = es.Search(index=config.elastic.index_name)

nomad/cli/client/statistics.py

@@ -34,7 +34,7 @@ def codes(client, minimum=1, **kwargs):
     data = client.repo.search(per_page=1, **kwargs).response().result
 
     x_values = sorted([
-        code for code, values in data.statistics['code_name'].items()
+        code for code, values in data.statistics['dft.code_name'].items()
         if code != 'not processed' and values.get('calculations', 1000) >= minimum], key=lambda x: x.lower())
 
     return data.statistics, x_values, 'code_name', 'code'
@@ -61,7 +61,7 @@ def error_fig(client):
 
         return {
             code: values[metric]
-            for code, values in result.quantities['code_name'].items()
+            for code, values in result.quantities['dft.code_name'].items()
             if code != 'not processed' and (not labels or code in labels) > 0}
 
     # get the data
@@ -414,7 +414,7 @@ def statistics_table(html, geometries, public_path):
         'total', 'all', 'code_runs')
 
     phonons = get_statistic(
-        client.repo.search(per_page=1, code_name='Phonopy').response().result,
+        client.repo.search(per_page=1, **{'dft.code_name': 'Phonopy'}).response().result,
         'total', 'all', 'code_runs')
 
     # files and sized

nomad/datamodel/dft.py

@@ -335,4 +335,4 @@ Domain(
     groups=dict(
         groups=('group_hash', 'groups')),
     default_statistics=[
-        'atoms', 'basis_set', 'xc_functional', 'system', 'crystal_system', 'code_name'])
+        'dft.atoms', 'dft.basis_set', 'dft.xc_functional', 'dft.system', 'dft.crystal_system', 'dft.code_name'])

tests/app/test_api.py

@@ -815,7 +815,7 @@ class TestRepo():
         data = self.assert_search(rv, calcs)
         results = data.get('results', None)
         if calcs > 0:
-            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
+            for key in ['uploader', 'calc_id', 'dft.formula', 'upload_id']:
                 assert key in results[0]
 
     @pytest.mark.parametrize('calcs, start, end', [
@@ -844,22 +844,22 @@ class TestRepo():
         self.assert_search(rv, calcs)
 
     @pytest.mark.parametrize('calcs, quantity, value, user', [
-        (2, 'system', 'bulk', 'test_user'),
-        (0, 'system', 'atom', 'test_user'),
-        (1, 'atoms', 'Br', 'test_user'),
-        (1, 'atoms', 'Fe', 'test_user'),
-        (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
-        (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
-        (1, 'only_atoms', ['Fe'], 'test_user'),
-        (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
-        (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
+        (2, 'dft.system', 'bulk', 'test_user'),
+        (0, 'dft.system', 'atom', 'test_user'),
+        (1, 'dft.atoms', 'Br', 'test_user'),
+        (1, 'dft.atoms', 'Fe', 'test_user'),
+        (0, 'dft.atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
+        (0, 'dft.only_atoms', ['Br', 'Si'], 'test_user'),
+        (1, 'dft.only_atoms', ['Fe'], 'test_user'),
+        (1, 'dft.only_atoms', ['Br', 'K', 'Si'], 'test_user'),
+        (1, 'dft.only_atoms', ['Br', 'Si', 'K'], 'test_user'),
         (1, 'comment', 'specific', 'test_user'),
         (1, 'authors', 'Leonard Hofstadter', 'test_user'),
         (2, 'files', 'test/mainfile.txt', 'test_user'),
         (2, 'paths', 'mainfile.txt', 'test_user'),
         (2, 'paths', 'test', 'test_user'),
-        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
-        (0, 'quantities', 'dos', 'test_user'),
+        (2, 'dft.quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
+        (0, 'dft.quantities', 'dos', 'test_user'),
         (2, 'external_id', 'external_2,external_3', 'other_test_user'),
         (1, 'external_id', 'external_2', 'test_user'),
         (1, 'external_id', 'external_2,external_3', 'test_user'),
@@ -879,17 +879,17 @@ class TestRepo():
         assert statistics is not None
         if quantity == 'system' and calcs != 0:
             # for simplicity we only assert on quantities for this case
-            assert 'system' in statistics
-            assert len(statistics['system']) == 1
-            assert value in statistics['system']
+            assert 'dft.system' in statistics
+            assert len(statistics['dft.system']) == 1
+            assert value in statistics['dft.system']
 
     def test_search_exclude(self, api, example_elastic_calcs, no_warn):
-        rv = api.get('/repo/?exclude=atoms,only_atoms')
+        rv = api.get('/repo/?exclude=dft.atoms,dft.only_atoms')
         assert rv.status_code == 200
         result = json.loads(rv.data)['results'][0]
-        assert 'atoms' not in result
-        assert 'only_atoms' not in result
-        assert 'basis_set' in result
+        assert 'dft.atoms' not in result
+        assert 'dft.only_atoms' not in result
+        assert 'dft.basis_set' in result
 
     metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]
 
@@ -950,8 +950,8 @@ class TestRepo():
         assert len(results) == n_results
 
     @pytest.mark.parametrize('first, order_by, order', [
-        ('1', 'formula', -1), ('2', 'formula', 1),
-        ('2', 'basis_set', -1), ('1', 'basis_set', 1),
+        ('1', 'dft.formula', -1), ('2', 'dft.formula', 1),
+        ('2', 'dft.basis_set', -1), ('1', 'dft.basis_set', 1),
         (None, 'authors', -1)])
     def test_search_order(self, api, example_elastic_calcs, no_warn, first, order_by, order):
         rv = api.get('/repo/?order_by=%s&order=%d' % (order_by, order))
@@ -994,13 +994,13 @@ class TestRepo():
         assert rv.status_code == 401
 
     @pytest.mark.parametrize('calcs, quantity, value', [
-        (2, 'system', 'bulk'),
-        (0, 'system', 'atom'),
-        (1, 'atoms', 'Br'),
-        (1, 'atoms', 'Fe'),
+        (2, 'dft.system', 'bulk'),
+        (0, 'dft.system', 'atom'),
+        (1, 'dft.atoms', 'Br'),
+        (1, 'dft.atoms', 'Fe'),
         (1, 'authors', 'Leonard Hofstadter'),
         (2, 'files', 'test/mainfile.txt'),
-        (0, 'quantities', 'dos')
+        (0, 'dft.quantities', 'dos')
     ])
     def test_quantity_search(self, api, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
         rv = api.get('/repo/quantity/%s' % quantity, headers=test_user_auth)
@@ -1014,7 +1014,7 @@ class TestRepo():
             assert 0 == calcs
 
     def test_quantity_search_after(self, api, example_elastic_calcs, no_warn, test_user_auth):
-        rv = api.get('/repo/quantity/atoms?size=1')
+        rv = api.get('/repo/quantity/dft.atoms?size=1')
         assert rv.status_code == 200
         data = json.loads(rv.data)
 
@@ -1025,7 +1025,7 @@ class TestRepo():
         value = list(quantity['values'].keys())[0]
 
         while True:
-            rv = api.get('/repo/quantity/atoms?size=1&after=%s' % after)
+            rv = api.get('/repo/quantity/dft.atoms?size=1&after=%s' % after)
             assert rv.status_code == 200
             data = json.loads(rv.data)
 
@@ -1040,7 +1040,10 @@ class TestRepo():
             after = quantity['after']
 
     def test_quantities_search(self, api, example_elastic_calcs, no_warn, test_user_auth):
-        rv = api.get('/repo/quantities?%s' % urlencode(dict(quantities=['system', 'atoms'], size=1), doseq=True), headers=test_user_auth)
+        rv = api.get(
+            '/repo/quantities?%s' % urlencode(
+                dict(quantities=['dft.system', 'dft.atoms'], size=1), doseq=True),
+            headers=test_user_auth)
         assert rv.status_code == 200
         # TODO actual assertions
 

tests/app/test_app.py

@@ -52,7 +52,7 @@ def test_internal_server_error_get(client, caplog):
     rv = client.get('/api/test/ise?test_arg=value')
     assert rv.status_code == 500
     record = assert_log(caplog, 'error', 'internal server error')
-    data = json.loads(record.message)
+    data = json.loads(record.msg)
 
     assert data['blueprint'] == 'api'
     assert data['endpoint'] == 'api.test_internal_server_error_resource'
@@ -67,7 +67,7 @@ def test_internal_server_error_post(client, caplog):
         data=json.dumps(dict(test_arg='value')))
     assert rv.status_code == 500
     record = assert_log(caplog, 'error', 'internal server error')
-    data = json.loads(record.message)
+    data = json.loads(record.msg)
 
     assert data['blueprint'] == 'api'
     assert data['endpoint'] == 'api.test_internal_server_error_resource'

tests/app/test_optimade.py

@@ -36,7 +36,7 @@ def test_get_entry(published: Upload):
         data = json.load(f)
     assert 'OptimadeEntry' in data
     search_result = search.SearchRequest().search_parameter('calc_id', calc_id).execute_paginated()['results'][0]
-    assert 'optimade' in search_result
+    assert 'dft.optimade' in search_result
 
 
 def test_no_optimade(meta_info, elastic, api):

tests/conftest.py

@@ -49,7 +49,7 @@ example_files = [empty_file, example_file]
 
 
 utils.ConsoleFormatter.short_format = True
-logging.Formatter = utils.ConsoleFormatter
+setattr(logging, 'Formatter', utils.ConsoleFormatter)
 
 
 @pytest.fixture(scope="session")

tests/processing/test_data.py

@@ -383,8 +383,6 @@ def test_malicious_parser_task_failure(proc_infra, failure, test_user):
 
 
 def test_ems_data(proc_infra, test_user):
-
-
     upload = run_processing(('test_ems_upload', 'tests/data/proc/example_ems.zip'), test_user)
 
     additional_keys = [

tests/test_cli.py

@@ -279,7 +279,7 @@ class TestClient:
 
         new_search_results = new_search['results'][0]
         for key in new_search_results.keys():
-            if key not in ['upload_time', 'last_processing', 'labels']:
+            if key not in ['upload_time', 'last_processing', 'dft.labels']:
                 # There is a sub second change due to date conversions (?).
                 # Labels have arbitrary order.
                 assert json.dumps(new_search_results[key]) == json.dumps(ref_search_results[key])