Added search functions back.

.gitmodules

@@ -41,4 +41,7 @@
 [submodule "dependencies/parsers/wien2k"]
 	path = dependencies/parsers/wien2k
 	url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-wien2k
-	branch = nomad-fair
\ No newline at end of file
+	branch = nomad-fair
+[submodule "dependencies/parsers/parser-band"]
+	path = dependencies/parsers/band
+	url = git@gitlab.mpcdf.mpg.de:nomad-lab/parser-band.git

.vscode/launch.json

@@ -44,7 +44,7 @@
       "cwd": "${workspaceFolder}",
       "program": "${workspaceFolder}/.pyenv/bin/pytest",
       "args": [
-        "-sv", "tests/test_parsing.py::test_parser[parsers/vasp-tests/data/parsers/vasp_compressed/vasp.xml.gz]"
+        "-sv", "tests/test_api.py::TestRepo::test_search[2-user-other_test_user]"
       ]
     },
     {

band @ 2e47c739

+Subproject commit 2e47c739d555088c51900fbbe973b86e6b1e603b

gui/src/components/Repo.js

@@ -49,13 +49,13 @@ class Repo extends React.Component {
   })
 
   static rowConfig = {
-    chemical_composition: 'Formula',
-    program_name: 'Code',
-    basis_set_type: 'Basis set',
-    system_type: 'System',
-    crystal_system: 'Crystal',
-    space_group_number: 'Space group',
-    XC_functional_name: 'XT treatment'
+    formula: 'Formula',
+    code_name: 'Code',
+    basis_set: 'Basis set',
+    system: 'System',
+    crystal_system: 'Crystal system',
+    spacegroup: 'Spacegroup',
+    xc_functional: 'XT treatment'
   }
 
   state = {

nomad/api/repo.py

@@ -18,8 +18,11 @@ meta-data.
 """
 
 from flask_restplus import Resource, abort, fields
+from flask import request, g
+from elasticsearch_dsl import Q
 
 from nomad.files import UploadFiles, Restricted
+from nomad.search import Entry
 
 from .app import api
 from .auth import login_if_available, create_authorization_predicate
@@ -80,38 +83,42 @@ class RepoCalcsResource(Resource):
 
         This is currently not implemented!
         """
-        return dict(pagination=dict(total=0, page=1, per_page=10), results=[]), 200
-        # page = int(request.args.get('page', 1))
-        # per_page = int(request.args.get('per_page', 10))
-        # owner = request.args.get('owner', 'all')
-
-        # try:
-        #     assert page >= 1
-        #     assert per_page > 0
-        # except AssertionError:
-        #     abort(400, message='invalid pagination')
-
-        # if owner == 'all':
-        #     search = RepoCalc.search().query('match_all')
-        # elif owner == 'user':
-        #     if g.user is None:
-        #         abort(401, message='Authentication required for owner value user.')
-        #     search = RepoCalc.search().query('match_all')
-        #     search = search.filter('term', user_id=str(g.user.user_id))
-        # elif owner == 'staging':
-        #     if g.user is None:
-        #         abort(401, message='Authentication required for owner value user.')
-        #     search = RepoCalc.search().query('match_all')
-        #     search = search.filter('term', user_id=str(g.user.user_id)).filter('term', staging=True)
-        # else:
-        #     abort(400, message='Invalid owner value. Valid values are all|user|staging, default is all')
-
-        # search = search[(page - 1) * per_page: page * per_page]
-        # return {
-        #     'pagination': {
-        #         'total': search.count(),
-        #         'page': page,
-        #         'per_page': per_page
-        #     },
-        #     'results': [result.json_dict for result in search]
-        # }, 200
+        # return dict(pagination=dict(total=0, page=1, per_page=10), results=[]), 200
+
+        page = int(request.args.get('page', 1))
+        per_page = int(request.args.get('per_page', 10))
+        owner = request.args.get('owner', 'all')
+
+        try:
+            assert page >= 1
+            assert per_page > 0
+        except AssertionError:
+            abort(400, message='invalid pagination')
+
+        if owner == 'all':
+            if g.user is None:
+                q = Q('term', published=True)
+            else:
+                q = Q('term', published=True) | Q('term', uploader__user_id=g.user.user_id)
+        elif owner == 'user':
+            if g.user is None:
+                abort(401, message='Authentication required for owner value user.')
+
+            q = Q('term', uploader__user_id=g.user.user_id)
+        elif owner == 'staging':
+            if g.user is None:
+                abort(401, message='Authentication required for owner value user.')
+            q = Q('term', published=False) & Q('term', uploader__user_id=g.user.user_id)
+        else:
+            abort(400, message='Invalid owner value. Valid values are all|user|staging, default is all')
+
+        search = Entry.search().query(q)
+        search = search[(page - 1) * per_page: page * per_page]
+        return {
+            'pagination': {
+                'total': search.count(),
+                'page': page,
+                'per_page': per_page
+            },
+            'results': [hit.to_dict() for hit in search]
+        }, 200

nomad/datamodel.py

@@ -98,6 +98,7 @@ class CalcWithMetadata():
         self.uploader: utils.POPO = None
 
         self.with_embargo: bool = None
+        self.published: bool = False
         self.coauthors: List[utils.POPO] = []
         self.shared_with: List[utils.POPO] = []
         self.comment: str = None
@@ -114,8 +115,7 @@ class CalcWithMetadata():
         self.code_name: str = None
         self.code_version: str = None
 
-        for key, value in kwargs.items():
-            setattr(self, key, value)
+        self.update(**kwargs)
 
     def to_dict(self):
         return {
@@ -123,6 +123,10 @@ class CalcWithMetadata():
             if value is not None
         }
 
+    def update(self, **kwargs):
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
     def apply_user_metadata(self, metadata: dict):
         """
         Applies a user provided metadata dict to this calc.

nomad/processing/data.py

@@ -239,7 +239,8 @@ class Calc(Proc):
 
         # index in search
         with utils.timer(logger, 'indexed', step='persist'):
-            search.Entry.from_calc_with_metadata(calc_with_metadata, published=False).save()
+            calc_with_metadata.update(published=False, uploader=self.upload.uploader.to_popo())
+            search.Entry.from_calc_with_metadata(calc_with_metadata).save()
 
         # persist the archive
         with utils.timer(

nomad/search.py

@@ -17,7 +17,7 @@ This module represents calculations in elastic search.
 """
 
 from elasticsearch_dsl import Document, InnerDoc, Keyword, Text, Date, \
-    Nested, Boolean, Search
+    Object, Boolean, Search
 
 from nomad import config, datamodel, infrastructure, datamodel, coe_repo
 
@@ -29,7 +29,7 @@ class User(InnerDoc):
 
     @classmethod
     def from_user_popo(cls, user):
-        self = cls(id=user.id)
+        self = cls(user_id=user.id)
 
         if 'first_name' not in user:
             user = coe_repo.User.from_user_id(user.id).to_popo()
@@ -39,7 +39,7 @@ class User(InnerDoc):
 
         return self
 
-    id = Keyword()
+    user_id = Keyword()
     name = Text()
     name_keyword = Keyword()
 
@@ -69,16 +69,16 @@ class Entry(Document):
     pid = Keyword()
     mainfile = Keyword()
     files = Keyword(multi=True)
-    uploader = Nested(User)
+    uploader = Object(User)
 
     with_embargo = Boolean()
     published = Boolean()
 
-    coauthors = Nested(User)
-    shared_with = Nested(User)
+    coauthors = Object(User)
+    shared_with = Object(User)
     comment = Text()
     references = Keyword()
-    datasets = Nested(Dataset)
+    datasets = Object(Dataset)
 
     formula = Keyword()
     atoms = Keyword(multi=True)
@@ -91,7 +91,7 @@ class Entry(Document):
     code_version = Keyword()
 
     @classmethod
-    def from_calc_with_metadata(cls, source: datamodel.CalcWithMetadata, published: bool = False) -> 'Entry':
+    def from_calc_with_metadata(cls, source: datamodel.CalcWithMetadata) -> 'Entry':
         return Entry(
             meta=dict(id=source.calc_id),
             upload_id=source.upload_id,
@@ -104,7 +104,7 @@ class Entry(Document):
             uploader=User.from_user_popo(source.uploader) if source.uploader is not None else None,
 
             with_embargo=source.with_embargo,
-            published=published,
+            published=source.published,
             coauthors=[User.from_user_popo(user) for user in source.coauthors],
             shared_with=[User.from_user_popo(user) for user in source.shared_with],
             comment=source.comment,

tests/conftest.py

@@ -36,6 +36,7 @@ from tests.processing import test_data as test_processing
 from tests.test_files import example_file, empty_file
 from tests.bravado_flask import FlaskTestHttpClient
 
+test_log_level = logging.CRITICAL
 example_files = [empty_file, example_file]
 
 
@@ -50,7 +51,7 @@ def monkeysession(request):
 @pytest.fixture(scope='session', autouse=True)
 def nomad_logging():
     config.logstash = config.logstash._replace(enabled=False)
-    config.console_log_level = logging.CRITICAL
+    config.console_log_level = test_log_level
     infrastructure.setup_logging()
 
 
@@ -129,7 +130,7 @@ def celery_inspect(purged_app):
 
 # It might be necessary to make this a function scoped fixture, if old tasks keep
 # 'bleeding' into successive tests.
-@pytest.fixture(scope='session')
+@pytest.fixture(scope='function')
 def worker(celery_session_worker, celery_inspect):
     """ Provides a clean worker (no old tasks) per function. Waits for all tasks to be completed. """
     pass
@@ -170,18 +171,22 @@ def elastic_infra(monkeysession):
         return infrastructure.setup_elastic()
 
 
-@pytest.fixture(scope='function')
-def elastic(elastic_infra):
-    """ Provides a clean elastic per function. Clears elastic before test. """
+def clear_elastic(elastic):
     while True:
         try:
-            elastic_infra.delete_by_query(
+            elastic.delete_by_query(
                 index='test_nomad_fairdi_calcs', body=dict(query=dict(match_all={})),
                 wait_for_completion=True, refresh=True)
             break
         except Exception:
             time.sleep(0.1)
 
+
+@pytest.fixture(scope='function')
+def elastic(elastic_infra):
+    """ Provides a clean elastic per function. Clears elastic before test. """
+    clear_elastic(elastic_infra)
+
     assert infrastructure.elastic_client is not None
     return elastic_infra
 
@@ -302,7 +307,7 @@ def test_user_auth(test_user: coe_repo.User):
 
 
 @pytest.fixture(scope='module')
-def test_other_user_auth(other_test_user: coe_repo.User):
+def other_test_user_auth(other_test_user: coe_repo.User):
     return create_auth_headers(other_test_user)
 
 
@@ -483,14 +488,14 @@ def example_user_metadata(other_test_user, test_user) -> dict:
     }
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture(scope='session')
 def parsed(example_mainfile: Tuple[str, str]) -> parsing.LocalBackend:
     """ Provides a parsed calculation in the form of a LocalBackend. """
     parser, mainfile = example_mainfile
     return test_parsing.run_parser(parser, mainfile)
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture(scope='session')
 def normalized(parsed: parsing.LocalBackend) -> parsing.LocalBackend:
     """ Provides a normalized calculation in the form of a LocalBackend. """
     return test_normalizing.run_normalize(parsed)

tests/test_api.py

@@ -21,12 +21,11 @@ import io
 import inspect
 from passlib.hash import bcrypt
 
-from nomad import config, coe_repo
+from nomad import config, coe_repo, search, parsing
 from nomad.files import UploadFiles, PublicUploadFiles
 from nomad.processing import Upload, Calc, SUCCESS
-from nomad.coe_repo import User
 
-from tests.conftest import create_auth_headers
+from tests.conftest import create_auth_headers, clear_elastic
 from tests.test_files import example_file, example_file_mainfile, example_file_contents
 from tests.test_files import create_staging_upload, create_public_upload
 from tests.test_coe_repo import assert_coe_upload
@@ -85,7 +84,7 @@ class TestAdmin:
 
 
 class TestAuth:
-    def test_xtoken_auth(self, client, test_user: User, no_warn):
+    def test_xtoken_auth(self, client, test_user: coe_repo.User, no_warn):
         rv = client.get('/uploads/', headers={
             'X-Token': test_user.first_name.lower()  # the test users have their firstname as tokens for convinience
         })
@@ -110,7 +109,7 @@ class TestAuth:
         })
         assert rv.status_code == 401
 
-    def test_get_user(self, client, test_user_auth, test_user: User, no_warn):
+    def test_get_user(self, client, test_user_auth, test_user: coe_repo.User, no_warn):
         rv = client.get('/auth/user', headers=test_user_auth)
         assert rv.status_code == 200
         self.assert_user(client, json.loads(rv.data))
@@ -521,6 +520,24 @@ class TestArchive(UploadFilesBasedTests):
 
 
 class TestRepo(UploadFilesBasedTests):
+    @pytest.fixture(scope='class')
+    def example_elastic_calcs(
+            self, elastic_infra, normalized: parsing.LocalBackend,
+            test_user: coe_repo.User, other_test_user: coe_repo.User):
+
+        clear_elastic(elastic_infra)
+
+        calc_with_metadata = normalized.to_calc_with_metadata()
+
+        calc_with_metadata.update(calc_id='1', uploader=test_user.to_popo(), published=True)
+        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
+
+        calc_with_metadata.update(calc_id='2', uploader=other_test_user.to_popo(), published=True)
+        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
+
+        calc_with_metadata.update(calc_id='3', uploader=other_test_user.to_popo(), published=False)
+        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
+
     @UploadFilesBasedTests.ignore_authorization
     def test_calc(self, client, upload, auth_headers):
         rv = client.get('/repo/%s/0' % upload, headers=auth_headers)
@@ -531,43 +548,39 @@ class TestRepo(UploadFilesBasedTests):
         rv = client.get('/repo/doesnt/exist', headers=auth_headers)
         assert rv.status_code == 404
 
-    # def test_calcs(self, client, example_elastic_calc, no_warn):
-    #     rv = client.get('/repo/')
-    #     assert rv.status_code == 200
-    #     data = json.loads(rv.data)
-    #     results = data.get('results', None)
-    #     assert results is not None
-    #     assert isinstance(results, list)
-    #     assert len(results) >= 1
-
-    # def test_calcs_pagination(self, client, example_elastic_calc, no_warn):
-    #     rv = client.get('/repo/?page=1&per_page=1')
-    #     assert rv.status_code == 200
-    #     data = json.loads(rv.data)
-    #     results = data.get('results', None)
-    #     assert results is not None
-    #     assert isinstance(results, list)
-    #     assert len(results) == 1
-
-    # def test_calcs_user(self, client, example_elastic_calc, test_user_auth, no_warn):
-    #     rv = client.get('/repo/?owner=user', headers=test_user_auth)
-    #     assert rv.status_code == 200
-    #     data = json.loads(rv.data)
-    #     results = data.get('results', None)
-    #     assert results is not None
-    #     assert len(results) >= 1
-
-    # def test_calcs_user_authrequired(self, client, example_elastic_calc, no_warn):
-    #     rv = client.get('/repo/?owner=user')
-    #     assert rv.status_code == 401
-
-    # def test_calcs_user_invisible(self, client, example_elastic_calc, test_other_user_auth, no_warn):
-    #     rv = client.get('/repo/?owner=user', headers=test_other_user_auth)
-    #     assert rv.status_code == 200
-    #     data = json.loads(rv.data)
-    #     results = data.get('results', None)
-    #     assert results is not None
-    #     assert len(results) == 0
+    @pytest.mark.parametrize('calcs, owner, auth', [
+        (2, 'all', 'none'),
+        (2, 'all', 'test_user'),
+        (1, 'user', 'test_user'),
+        (2, 'user', 'other_test_user'),
+        (0, 'staging', 'test_user'),
+        (1, 'staging', 'other_test_user'),
+    ])
+    def test_search(self, client, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
+        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
+        rv = client.get('/repo/?owner=%s' % owner, headers=auth)
+        assert rv.status_code == 200
+        data = json.loads(rv.data)
+        results = data.get('results', None)
+        assert results is not None
+        assert isinstance(results, list)
+        assert len(results) == calcs
+        if calcs > 0:
+            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
+                assert key in results[0]
+
+    def test_calcs_pagination(self, client, example_elastic_calcs, no_warn):
+        rv = client.get('/repo/?page=1&per_page=1')
+        assert rv.status_code == 200
+        data = json.loads(rv.data)
+        results = data.get('results', None)
+        assert results is not None
+        assert isinstance(results, list)
+        assert len(results) == 1
+
+    def test_search_user_authrequired(self, client, example_elastic_calcs, no_warn):
+        rv = client.get('/repo/?owner=user')
+        assert rv.status_code == 401
 
 
 class TestRaw(UploadFilesBasedTests):

tests/test_search.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from elasticsearch_dsl import Q
+
 from nomad import datamodel, search, processing, parsing
 from nomad.search import Entry
 
@@ -46,10 +48,15 @@ def test_index_upload(elastic, processed: processing.Upload):
 
 
 def create_entry(calc_with_metadata: datamodel.CalcWithMetadata):
-    search.Entry.from_calc_with_metadata(calc_with_metadata).save()
+    search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
     assert_entry(calc_with_metadata.calc_id)
 
 
 def assert_entry(calc_id):
     calc = Entry.get(calc_id)
     assert calc is not None
+
+    search = Entry.search().query(Q('term', calc_id=calc_id))[0:10]
+    assert search.count() == 1
+    results = list(hit.to_dict() for hit in search)
+    assert results[0]['calc_id'] == calc_id