diff --git a/.gitmodules b/.gitmodules
index c3ac1c7f45b8c0b275af49c6f826df90849ab71d..324812f5ee189c3ddf42b5ead52f542ea9fec75d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -41,4 +41,7 @@
[submodule "dependencies/parsers/wien2k"]
path = dependencies/parsers/wien2k
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-wien2k
- branch = nomad-fair
\ No newline at end of file
+ branch = nomad-fair
+[submodule "dependencies/parsers/parser-band"]
+ path = dependencies/parsers/band
+ url = git@gitlab.mpcdf.mpg.de:nomad-lab/parser-band.git
diff --git a/.vscode/launch.json b/.vscode/launch.json
index e832da43012039853b691d44a2f87a40beca9e64..8c7a98bc22c8ff21efb69144a37194bb133dc1e7 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -44,7 +44,7 @@
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
- "-sv", "tests/test_parsing.py::test_parser[parsers/vasp-tests/data/parsers/vasp_compressed/vasp.xml.gz]"
+ "-sv", "tests/test_api.py::TestRepo::test_search[2-user-other_test_user]"
]
},
{
diff --git a/dependencies/parsers/band b/dependencies/parsers/band
new file mode 160000
index 0000000000000000000000000000000000000000..2e47c739d555088c51900fbbe973b86e6b1e603b
--- /dev/null
+++ b/dependencies/parsers/band
@@ -0,0 +1 @@
+Subproject commit 2e47c739d555088c51900fbbe973b86e6b1e603b
diff --git a/gui/src/components/Repo.js b/gui/src/components/Repo.js
index ba3e5fbffd99767210defdf7a9b2e759b2cf4c15..c36252ce1c887b44943d1ffa38bfd7a74bdc2a9f 100644
--- a/gui/src/components/Repo.js
+++ b/gui/src/components/Repo.js
@@ -49,13 +49,13 @@ class Repo extends React.Component {
})
static rowConfig = {
- chemical_composition: 'Formula',
- program_name: 'Code',
- basis_set_type: 'Basis set',
- system_type: 'System',
- crystal_system: 'Crystal',
- space_group_number: 'Space group',
- XC_functional_name: 'XT treatment'
+ formula: 'Formula',
+ code_name: 'Code',
+ basis_set: 'Basis set',
+ system: 'System',
+ crystal_system: 'Crystal system',
+ spacegroup: 'Spacegroup',
+ xc_functional: 'XT treatment'
}
state = {
diff --git a/nomad/api/repo.py b/nomad/api/repo.py
index 7131715ee52d1f937bb8df188b7fa34412ca3d3b..292ba8eecb56e04cf0eb70b384a7b7eb2883b672 100644
--- a/nomad/api/repo.py
+++ b/nomad/api/repo.py
@@ -18,8 +18,11 @@ meta-data.
"""
from flask_restplus import Resource, abort, fields
+from flask import request, g
+from elasticsearch_dsl import Q
from nomad.files import UploadFiles, Restricted
+from nomad.search import Entry
from .app import api
from .auth import login_if_available, create_authorization_predicate
@@ -80,38 +83,42 @@ class RepoCalcsResource(Resource):
This is currently not implemented!
"""
- return dict(pagination=dict(total=0, page=1, per_page=10), results=[]), 200
- # page = int(request.args.get('page', 1))
- # per_page = int(request.args.get('per_page', 10))
- # owner = request.args.get('owner', 'all')
-
- # try:
- # assert page >= 1
- # assert per_page > 0
- # except AssertionError:
- # abort(400, message='invalid pagination')
-
- # if owner == 'all':
- # search = RepoCalc.search().query('match_all')
- # elif owner == 'user':
- # if g.user is None:
- # abort(401, message='Authentication required for owner value user.')
- # search = RepoCalc.search().query('match_all')
- # search = search.filter('term', user_id=str(g.user.user_id))
- # elif owner == 'staging':
- # if g.user is None:
- # abort(401, message='Authentication required for owner value user.')
- # search = RepoCalc.search().query('match_all')
- # search = search.filter('term', user_id=str(g.user.user_id)).filter('term', staging=True)
- # else:
- # abort(400, message='Invalid owner value. Valid values are all|user|staging, default is all')
-
- # search = search[(page - 1) * per_page: page * per_page]
- # return {
- # 'pagination': {
- # 'total': search.count(),
- # 'page': page,
- # 'per_page': per_page
- # },
- # 'results': [result.json_dict for result in search]
- # }, 200
+ # return dict(pagination=dict(total=0, page=1, per_page=10), results=[]), 200
+
+ page = int(request.args.get('page', 1))
+ per_page = int(request.args.get('per_page', 10))
+ owner = request.args.get('owner', 'all')
+
+ try:
+ assert page >= 1
+ assert per_page > 0
+ except AssertionError:
+ abort(400, message='invalid pagination')
+
+ if owner == 'all':
+ if g.user is None:
+ q = Q('term', published=True)
+ else:
+ q = Q('term', published=True) | Q('term', uploader__user_id=g.user.user_id)
+ elif owner == 'user':
+ if g.user is None:
+ abort(401, message='Authentication required for owner value user.')
+
+ q = Q('term', uploader__user_id=g.user.user_id)
+ elif owner == 'staging':
+ if g.user is None:
+ abort(401, message='Authentication required for owner value user.')
+ q = Q('term', published=False) & Q('term', uploader__user_id=g.user.user_id)
+ else:
+ abort(400, message='Invalid owner value. Valid values are all|user|staging, default is all')
+
+ search = Entry.search().query(q)
+ search = search[(page - 1) * per_page: page * per_page]
+ return {
+ 'pagination': {
+ 'total': search.count(),
+ 'page': page,
+ 'per_page': per_page
+ },
+ 'results': [hit.to_dict() for hit in search]
+ }, 200
diff --git a/nomad/datamodel.py b/nomad/datamodel.py
index 2df4069f63b260a9130cb2da3bb98f924d19fc52..edbcee639363203c3cd3f41f1d8d8c8a371b14c8 100644
--- a/nomad/datamodel.py
+++ b/nomad/datamodel.py
@@ -98,6 +98,7 @@ class CalcWithMetadata():
self.uploader: utils.POPO = None
self.with_embargo: bool = None
+ self.published: bool = False
self.coauthors: List[utils.POPO] = []
self.shared_with: List[utils.POPO] = []
self.comment: str = None
@@ -114,8 +115,7 @@ class CalcWithMetadata():
self.code_name: str = None
self.code_version: str = None
- for key, value in kwargs.items():
- setattr(self, key, value)
+ self.update(**kwargs)
def to_dict(self):
return {
@@ -123,6 +123,10 @@ class CalcWithMetadata():
if value is not None
}
+ def update(self, **kwargs):
+ for key, value in kwargs.items():
+ setattr(self, key, value)
+
def apply_user_metadata(self, metadata: dict):
"""
Applies a user provided metadata dict to this calc.
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index 8d06aa9410e230bf6440a3c0c7deb47e5f37b90f..25c7bdf69193133d27db0a9f5b22d6738e1ff976 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -239,7 +239,8 @@ class Calc(Proc):
# index in search
with utils.timer(logger, 'indexed', step='persist'):
- search.Entry.from_calc_with_metadata(calc_with_metadata, published=False).save()
+ calc_with_metadata.update(published=False, uploader=self.upload.uploader.to_popo())
+ search.Entry.from_calc_with_metadata(calc_with_metadata).save()
# persist the archive
with utils.timer(
diff --git a/nomad/search.py b/nomad/search.py
index db5c79966b0a7ca186df94737c69806bebb6076b..b4572ccdc4d6629b3a0f69b7f29190c923cbb2ec 100644
--- a/nomad/search.py
+++ b/nomad/search.py
@@ -17,7 +17,7 @@ This module represents calculations in elastic search.
"""
from elasticsearch_dsl import Document, InnerDoc, Keyword, Text, Date, \
- Nested, Boolean, Search
+ Object, Boolean, Search
from nomad import config, datamodel, infrastructure, datamodel, coe_repo
@@ -29,7 +29,7 @@ class User(InnerDoc):
@classmethod
def from_user_popo(cls, user):
- self = cls(id=user.id)
+ self = cls(user_id=user.id)
if 'first_name' not in user:
user = coe_repo.User.from_user_id(user.id).to_popo()
@@ -39,7 +39,7 @@ class User(InnerDoc):
return self
- id = Keyword()
+ user_id = Keyword()
name = Text()
name_keyword = Keyword()
@@ -69,16 +69,16 @@ class Entry(Document):
pid = Keyword()
mainfile = Keyword()
files = Keyword(multi=True)
- uploader = Nested(User)
+ uploader = Object(User)
with_embargo = Boolean()
published = Boolean()
- coauthors = Nested(User)
- shared_with = Nested(User)
+ coauthors = Object(User)
+ shared_with = Object(User)
comment = Text()
references = Keyword()
- datasets = Nested(Dataset)
+ datasets = Object(Dataset)
formula = Keyword()
atoms = Keyword(multi=True)
@@ -91,7 +91,7 @@ class Entry(Document):
code_version = Keyword()
@classmethod
- def from_calc_with_metadata(cls, source: datamodel.CalcWithMetadata, published: bool = False) -> 'Entry':
+ def from_calc_with_metadata(cls, source: datamodel.CalcWithMetadata) -> 'Entry':
return Entry(
meta=dict(id=source.calc_id),
upload_id=source.upload_id,
@@ -104,7 +104,7 @@ class Entry(Document):
uploader=User.from_user_popo(source.uploader) if source.uploader is not None else None,
with_embargo=source.with_embargo,
- published=published,
+ published=source.published,
coauthors=[User.from_user_popo(user) for user in source.coauthors],
shared_with=[User.from_user_popo(user) for user in source.shared_with],
comment=source.comment,
diff --git a/tests/conftest.py b/tests/conftest.py
index 02350c7c5b706b10eed2e5f0d29d6178ca4b8007..314dca817cea1b4da2cc2600fa5b171ddbd7f8a1 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -36,6 +36,7 @@ from tests.processing import test_data as test_processing
from tests.test_files import example_file, empty_file
from tests.bravado_flask import FlaskTestHttpClient
+test_log_level = logging.CRITICAL
example_files = [empty_file, example_file]
@@ -50,7 +51,7 @@ def monkeysession(request):
@pytest.fixture(scope='session', autouse=True)
def nomad_logging():
config.logstash = config.logstash._replace(enabled=False)
- config.console_log_level = logging.CRITICAL
+ config.console_log_level = test_log_level
infrastructure.setup_logging()
@@ -129,7 +130,7 @@ def celery_inspect(purged_app):
# It might be necessary to make this a function scoped fixture, if old tasks keep
# 'bleeding' into successive tests.
-@pytest.fixture(scope='session')
+@pytest.fixture(scope='function')
def worker(celery_session_worker, celery_inspect):
""" Provides a clean worker (no old tasks) per function. Waits for all tasks to be completed. """
pass
@@ -170,18 +171,22 @@ def elastic_infra(monkeysession):
return infrastructure.setup_elastic()
-@pytest.fixture(scope='function')
-def elastic(elastic_infra):
- """ Provides a clean elastic per function. Clears elastic before test. """
+def clear_elastic(elastic):
while True:
try:
- elastic_infra.delete_by_query(
+ elastic.delete_by_query(
index='test_nomad_fairdi_calcs', body=dict(query=dict(match_all={})),
wait_for_completion=True, refresh=True)
break
except Exception:
time.sleep(0.1)
+
+@pytest.fixture(scope='function')
+def elastic(elastic_infra):
+ """ Provides a clean elastic per function. Clears elastic before test. """
+ clear_elastic(elastic_infra)
+
assert infrastructure.elastic_client is not None
return elastic_infra
@@ -302,7 +307,7 @@ def test_user_auth(test_user: coe_repo.User):
@pytest.fixture(scope='module')
-def test_other_user_auth(other_test_user: coe_repo.User):
+def other_test_user_auth(other_test_user: coe_repo.User):
return create_auth_headers(other_test_user)
@@ -483,14 +488,14 @@ def example_user_metadata(other_test_user, test_user) -> dict:
}
-@pytest.fixture(scope='function')
+@pytest.fixture(scope='session')
def parsed(example_mainfile: Tuple[str, str]) -> parsing.LocalBackend:
""" Provides a parsed calculation in the form of a LocalBackend. """
parser, mainfile = example_mainfile
return test_parsing.run_parser(parser, mainfile)
-@pytest.fixture(scope='function')
+@pytest.fixture(scope='session')
def normalized(parsed: parsing.LocalBackend) -> parsing.LocalBackend:
""" Provides a normalized calculation in the form of a LocalBackend. """
return test_normalizing.run_normalize(parsed)
diff --git a/tests/test_api.py b/tests/test_api.py
index 48e5c61aff6a5492b54f57c2a3fb51517dd3dcfa..47179e5115e992ecef608fa3513e4a75dec13435 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -21,12 +21,11 @@ import io
import inspect
from passlib.hash import bcrypt
-from nomad import config, coe_repo
+from nomad import config, coe_repo, search, parsing
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
-from nomad.coe_repo import User
-from tests.conftest import create_auth_headers
+from tests.conftest import create_auth_headers, clear_elastic
from tests.test_files import example_file, example_file_mainfile, example_file_contents
from tests.test_files import create_staging_upload, create_public_upload
from tests.test_coe_repo import assert_coe_upload
@@ -85,7 +84,7 @@ class TestAdmin:
class TestAuth:
- def test_xtoken_auth(self, client, test_user: User, no_warn):
+ def test_xtoken_auth(self, client, test_user: coe_repo.User, no_warn):
rv = client.get('/uploads/', headers={
'X-Token': test_user.first_name.lower() # the test users have their firstname as tokens for convinience
})
@@ -110,7 +109,7 @@ class TestAuth:
})
assert rv.status_code == 401
- def test_get_user(self, client, test_user_auth, test_user: User, no_warn):
+ def test_get_user(self, client, test_user_auth, test_user: coe_repo.User, no_warn):
rv = client.get('/auth/user', headers=test_user_auth)
assert rv.status_code == 200
self.assert_user(client, json.loads(rv.data))
@@ -521,6 +520,24 @@ class TestArchive(UploadFilesBasedTests):
class TestRepo(UploadFilesBasedTests):
+ @pytest.fixture(scope='class')
+ def example_elastic_calcs(
+ self, elastic_infra, normalized: parsing.LocalBackend,
+ test_user: coe_repo.User, other_test_user: coe_repo.User):
+
+ clear_elastic(elastic_infra)
+
+ calc_with_metadata = normalized.to_calc_with_metadata()
+
+ calc_with_metadata.update(calc_id='1', uploader=test_user.to_popo(), published=True)
+ search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
+
+ calc_with_metadata.update(calc_id='2', uploader=other_test_user.to_popo(), published=True)
+ search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
+
+ calc_with_metadata.update(calc_id='3', uploader=other_test_user.to_popo(), published=False)
+ search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
+
@UploadFilesBasedTests.ignore_authorization
def test_calc(self, client, upload, auth_headers):
rv = client.get('/repo/%s/0' % upload, headers=auth_headers)
@@ -531,43 +548,39 @@ class TestRepo(UploadFilesBasedTests):
rv = client.get('/repo/doesnt/exist', headers=auth_headers)
assert rv.status_code == 404
- # def test_calcs(self, client, example_elastic_calc, no_warn):
- # rv = client.get('/repo/')
- # assert rv.status_code == 200
- # data = json.loads(rv.data)
- # results = data.get('results', None)
- # assert results is not None
- # assert isinstance(results, list)
- # assert len(results) >= 1
-
- # def test_calcs_pagination(self, client, example_elastic_calc, no_warn):
- # rv = client.get('/repo/?page=1&per_page=1')
- # assert rv.status_code == 200
- # data = json.loads(rv.data)
- # results = data.get('results', None)
- # assert results is not None
- # assert isinstance(results, list)
- # assert len(results) == 1
-
- # def test_calcs_user(self, client, example_elastic_calc, test_user_auth, no_warn):
- # rv = client.get('/repo/?owner=user', headers=test_user_auth)
- # assert rv.status_code == 200
- # data = json.loads(rv.data)
- # results = data.get('results', None)
- # assert results is not None
- # assert len(results) >= 1
-
- # def test_calcs_user_authrequired(self, client, example_elastic_calc, no_warn):
- # rv = client.get('/repo/?owner=user')
- # assert rv.status_code == 401
-
- # def test_calcs_user_invisible(self, client, example_elastic_calc, test_other_user_auth, no_warn):
- # rv = client.get('/repo/?owner=user', headers=test_other_user_auth)
- # assert rv.status_code == 200
- # data = json.loads(rv.data)
- # results = data.get('results', None)
- # assert results is not None
- # assert len(results) == 0
+ @pytest.mark.parametrize('calcs, owner, auth', [
+ (2, 'all', 'none'),
+ (2, 'all', 'test_user'),
+ (1, 'user', 'test_user'),
+ (2, 'user', 'other_test_user'),
+ (0, 'staging', 'test_user'),
+ (1, 'staging', 'other_test_user'),
+ ])
+ def test_search(self, client, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
+ auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
+ rv = client.get('/repo/?owner=%s' % owner, headers=auth)
+ assert rv.status_code == 200
+ data = json.loads(rv.data)
+ results = data.get('results', None)
+ assert results is not None
+ assert isinstance(results, list)
+ assert len(results) == calcs
+ if calcs > 0:
+ for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
+ assert key in results[0]
+
+ def test_calcs_pagination(self, client, example_elastic_calcs, no_warn):
+ rv = client.get('/repo/?page=1&per_page=1')
+ assert rv.status_code == 200
+ data = json.loads(rv.data)
+ results = data.get('results', None)
+ assert results is not None
+ assert isinstance(results, list)
+ assert len(results) == 1
+
+ def test_search_user_authrequired(self, client, example_elastic_calcs, no_warn):
+ rv = client.get('/repo/?owner=user')
+ assert rv.status_code == 401
class TestRaw(UploadFilesBasedTests):
diff --git a/tests/test_search.py b/tests/test_search.py
index 8594182adc8d77febe11b79319268117578aaef8..2aeb618ca069dab8b715779878ebe1c2f7931eb6 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from elasticsearch_dsl import Q
+
from nomad import datamodel, search, processing, parsing
from nomad.search import Entry
@@ -46,10 +48,15 @@ def test_index_upload(elastic, processed: processing.Upload):
def create_entry(calc_with_metadata: datamodel.CalcWithMetadata):
- search.Entry.from_calc_with_metadata(calc_with_metadata).save()
+ search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
assert_entry(calc_with_metadata.calc_id)
def assert_entry(calc_id):
calc = Entry.get(calc_id)
assert calc is not None
+
+ search = Entry.search().query(Q('term', calc_id=calc_id))[0:10]
+ assert search.count() == 1
+ results = list(hit.to_dict() for hit in search)
+ assert results[0]['calc_id'] == calc_id