Commit 6b8340b7 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added search functions back.

parent 11cb9e74
......@@ -41,4 +41,7 @@
[submodule "dependencies/parsers/wien2k"]
path = dependencies/parsers/wien2k
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-wien2k
branch = nomad-fair
\ No newline at end of file
branch = nomad-fair
[submodule "dependencies/parsers/parser-band"]
path = dependencies/parsers/band
url = git@gitlab.mpcdf.mpg.de:nomad-lab/parser-band.git
......@@ -44,7 +44,7 @@
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_parsing.py::test_parser[parsers/vasp-tests/data/parsers/vasp_compressed/vasp.xml.gz]"
"-sv", "tests/test_api.py::TestRepo::test_search[2-user-other_test_user]"
]
},
{
......
Subproject commit 2e47c739d555088c51900fbbe973b86e6b1e603b
......@@ -49,13 +49,13 @@ class Repo extends React.Component {
})
static rowConfig = {
chemical_composition: 'Formula',
program_name: 'Code',
basis_set_type: 'Basis set',
system_type: 'System',
crystal_system: 'Crystal',
space_group_number: 'Space group',
XC_functional_name: 'XT treatment'
formula: 'Formula',
code_name: 'Code',
basis_set: 'Basis set',
system: 'System',
crystal_system: 'Crystal system',
spacegroup: 'Spacegroup',
xc_functional: 'XT treatment'
}
state = {
......
......@@ -18,8 +18,11 @@ meta-data.
"""
from flask_restplus import Resource, abort, fields
from flask import request, g
from elasticsearch_dsl import Q
from nomad.files import UploadFiles, Restricted
from nomad.search import Entry
from .app import api
from .auth import login_if_available, create_authorization_predicate
......@@ -80,38 +83,42 @@ class RepoCalcsResource(Resource):
This is currently not implemented!
"""
return dict(pagination=dict(total=0, page=1, per_page=10), results=[]), 200
# page = int(request.args.get('page', 1))
# per_page = int(request.args.get('per_page', 10))
# owner = request.args.get('owner', 'all')
# try:
# assert page >= 1
# assert per_page > 0
# except AssertionError:
# abort(400, message='invalid pagination')
# if owner == 'all':
# search = RepoCalc.search().query('match_all')
# elif owner == 'user':
# if g.user is None:
# abort(401, message='Authentication required for owner value user.')
# search = RepoCalc.search().query('match_all')
# search = search.filter('term', user_id=str(g.user.user_id))
# elif owner == 'staging':
# if g.user is None:
# abort(401, message='Authentication required for owner value user.')
# search = RepoCalc.search().query('match_all')
# search = search.filter('term', user_id=str(g.user.user_id)).filter('term', staging=True)
# else:
# abort(400, message='Invalid owner value. Valid values are all|user|staging, default is all')
# search = search[(page - 1) * per_page: page * per_page]
# return {
# 'pagination': {
# 'total': search.count(),
# 'page': page,
# 'per_page': per_page
# },
# 'results': [result.json_dict for result in search]
# }, 200
# return dict(pagination=dict(total=0, page=1, per_page=10), results=[]), 200
page = int(request.args.get('page', 1))
per_page = int(request.args.get('per_page', 10))
owner = request.args.get('owner', 'all')
try:
assert page >= 1
assert per_page > 0
except AssertionError:
abort(400, message='invalid pagination')
if owner == 'all':
if g.user is None:
q = Q('term', published=True)
else:
q = Q('term', published=True) | Q('term', uploader__user_id=g.user.user_id)
elif owner == 'user':
if g.user is None:
abort(401, message='Authentication required for owner value user.')
q = Q('term', uploader__user_id=g.user.user_id)
elif owner == 'staging':
if g.user is None:
abort(401, message='Authentication required for owner value user.')
q = Q('term', published=False) & Q('term', uploader__user_id=g.user.user_id)
else:
abort(400, message='Invalid owner value. Valid values are all|user|staging, default is all')
search = Entry.search().query(q)
search = search[(page - 1) * per_page: page * per_page]
return {
'pagination': {
'total': search.count(),
'page': page,
'per_page': per_page
},
'results': [hit.to_dict() for hit in search]
}, 200
......@@ -98,6 +98,7 @@ class CalcWithMetadata():
self.uploader: utils.POPO = None
self.with_embargo: bool = None
self.published: bool = False
self.coauthors: List[utils.POPO] = []
self.shared_with: List[utils.POPO] = []
self.comment: str = None
......@@ -114,8 +115,7 @@ class CalcWithMetadata():
self.code_name: str = None
self.code_version: str = None
for key, value in kwargs.items():
setattr(self, key, value)
self.update(**kwargs)
def to_dict(self):
return {
......@@ -123,6 +123,10 @@ class CalcWithMetadata():
if value is not None
}
def update(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
def apply_user_metadata(self, metadata: dict):
"""
Applies a user provided metadata dict to this calc.
......
......@@ -239,7 +239,8 @@ class Calc(Proc):
# index in search
with utils.timer(logger, 'indexed', step='persist'):
search.Entry.from_calc_with_metadata(calc_with_metadata, published=False).save()
calc_with_metadata.update(published=False, uploader=self.upload.uploader.to_popo())
search.Entry.from_calc_with_metadata(calc_with_metadata).save()
# persist the archive
with utils.timer(
......
......@@ -17,7 +17,7 @@ This module represents calculations in elastic search.
"""
from elasticsearch_dsl import Document, InnerDoc, Keyword, Text, Date, \
Nested, Boolean, Search
Object, Boolean, Search
from nomad import config, datamodel, infrastructure, datamodel, coe_repo
......@@ -29,7 +29,7 @@ class User(InnerDoc):
@classmethod
def from_user_popo(cls, user):
self = cls(id=user.id)
self = cls(user_id=user.id)
if 'first_name' not in user:
user = coe_repo.User.from_user_id(user.id).to_popo()
......@@ -39,7 +39,7 @@ class User(InnerDoc):
return self
id = Keyword()
user_id = Keyword()
name = Text()
name_keyword = Keyword()
......@@ -69,16 +69,16 @@ class Entry(Document):
pid = Keyword()
mainfile = Keyword()
files = Keyword(multi=True)
uploader = Nested(User)
uploader = Object(User)
with_embargo = Boolean()
published = Boolean()
coauthors = Nested(User)
shared_with = Nested(User)
coauthors = Object(User)
shared_with = Object(User)
comment = Text()
references = Keyword()
datasets = Nested(Dataset)
datasets = Object(Dataset)
formula = Keyword()
atoms = Keyword(multi=True)
......@@ -91,7 +91,7 @@ class Entry(Document):
code_version = Keyword()
@classmethod
def from_calc_with_metadata(cls, source: datamodel.CalcWithMetadata, published: bool = False) -> 'Entry':
def from_calc_with_metadata(cls, source: datamodel.CalcWithMetadata) -> 'Entry':
return Entry(
meta=dict(id=source.calc_id),
upload_id=source.upload_id,
......@@ -104,7 +104,7 @@ class Entry(Document):
uploader=User.from_user_popo(source.uploader) if source.uploader is not None else None,
with_embargo=source.with_embargo,
published=published,
published=source.published,
coauthors=[User.from_user_popo(user) for user in source.coauthors],
shared_with=[User.from_user_popo(user) for user in source.shared_with],
comment=source.comment,
......
......@@ -36,6 +36,7 @@ from tests.processing import test_data as test_processing
from tests.test_files import example_file, empty_file
from tests.bravado_flask import FlaskTestHttpClient
test_log_level = logging.CRITICAL
example_files = [empty_file, example_file]
......@@ -50,7 +51,7 @@ def monkeysession(request):
@pytest.fixture(scope='session', autouse=True)
def nomad_logging():
config.logstash = config.logstash._replace(enabled=False)
config.console_log_level = logging.CRITICAL
config.console_log_level = test_log_level
infrastructure.setup_logging()
......@@ -129,7 +130,7 @@ def celery_inspect(purged_app):
# It might be necessary to make this a function scoped fixture, if old tasks keep
# 'bleeding' into successive tests.
@pytest.fixture(scope='session')
@pytest.fixture(scope='function')
def worker(celery_session_worker, celery_inspect):
""" Provides a clean worker (no old tasks) per function. Waits for all tasks to be completed. """
pass
......@@ -170,18 +171,22 @@ def elastic_infra(monkeysession):
return infrastructure.setup_elastic()
@pytest.fixture(scope='function')
def elastic(elastic_infra):
""" Provides a clean elastic per function. Clears elastic before test. """
def clear_elastic(elastic):
while True:
try:
elastic_infra.delete_by_query(
elastic.delete_by_query(
index='test_nomad_fairdi_calcs', body=dict(query=dict(match_all={})),
wait_for_completion=True, refresh=True)
break
except Exception:
time.sleep(0.1)
@pytest.fixture(scope='function')
def elastic(elastic_infra):
""" Provides a clean elastic per function. Clears elastic before test. """
clear_elastic(elastic_infra)
assert infrastructure.elastic_client is not None
return elastic_infra
......@@ -302,7 +307,7 @@ def test_user_auth(test_user: coe_repo.User):
@pytest.fixture(scope='module')
def test_other_user_auth(other_test_user: coe_repo.User):
def other_test_user_auth(other_test_user: coe_repo.User):
return create_auth_headers(other_test_user)
......@@ -483,14 +488,14 @@ def example_user_metadata(other_test_user, test_user) -> dict:
}
@pytest.fixture(scope='function')
@pytest.fixture(scope='session')
def parsed(example_mainfile: Tuple[str, str]) -> parsing.LocalBackend:
""" Provides a parsed calculation in the form of a LocalBackend. """
parser, mainfile = example_mainfile
return test_parsing.run_parser(parser, mainfile)
@pytest.fixture(scope='function')
@pytest.fixture(scope='session')
def normalized(parsed: parsing.LocalBackend) -> parsing.LocalBackend:
""" Provides a normalized calculation in the form of a LocalBackend. """
return test_normalizing.run_normalize(parsed)
......
......@@ -21,12 +21,11 @@ import io
import inspect
from passlib.hash import bcrypt
from nomad import config, coe_repo
from nomad import config, coe_repo, search, parsing
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
from nomad.coe_repo import User
from tests.conftest import create_auth_headers
from tests.conftest import create_auth_headers, clear_elastic
from tests.test_files import example_file, example_file_mainfile, example_file_contents
from tests.test_files import create_staging_upload, create_public_upload
from tests.test_coe_repo import assert_coe_upload
......@@ -85,7 +84,7 @@ class TestAdmin:
class TestAuth:
def test_xtoken_auth(self, client, test_user: User, no_warn):
def test_xtoken_auth(self, client, test_user: coe_repo.User, no_warn):
rv = client.get('/uploads/', headers={
'X-Token': test_user.first_name.lower() # the test users have their firstname as tokens for convinience
})
......@@ -110,7 +109,7 @@ class TestAuth:
})
assert rv.status_code == 401
def test_get_user(self, client, test_user_auth, test_user: User, no_warn):
def test_get_user(self, client, test_user_auth, test_user: coe_repo.User, no_warn):
rv = client.get('/auth/user', headers=test_user_auth)
assert rv.status_code == 200
self.assert_user(client, json.loads(rv.data))
......@@ -521,6 +520,24 @@ class TestArchive(UploadFilesBasedTests):
class TestRepo(UploadFilesBasedTests):
@pytest.fixture(scope='class')
def example_elastic_calcs(
self, elastic_infra, normalized: parsing.LocalBackend,
test_user: coe_repo.User, other_test_user: coe_repo.User):
clear_elastic(elastic_infra)
calc_with_metadata = normalized.to_calc_with_metadata()
calc_with_metadata.update(calc_id='1', uploader=test_user.to_popo(), published=True)
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
calc_with_metadata.update(calc_id='2', uploader=other_test_user.to_popo(), published=True)
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
calc_with_metadata.update(calc_id='3', uploader=other_test_user.to_popo(), published=False)
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
@UploadFilesBasedTests.ignore_authorization
def test_calc(self, client, upload, auth_headers):
rv = client.get('/repo/%s/0' % upload, headers=auth_headers)
......@@ -531,43 +548,39 @@ class TestRepo(UploadFilesBasedTests):
rv = client.get('/repo/doesnt/exist', headers=auth_headers)
assert rv.status_code == 404
# def test_calcs(self, client, example_elastic_calc, no_warn):
# rv = client.get('/repo/')
# assert rv.status_code == 200
# data = json.loads(rv.data)
# results = data.get('results', None)
# assert results is not None
# assert isinstance(results, list)
# assert len(results) >= 1
# def test_calcs_pagination(self, client, example_elastic_calc, no_warn):
# rv = client.get('/repo/?page=1&per_page=1')
# assert rv.status_code == 200
# data = json.loads(rv.data)
# results = data.get('results', None)
# assert results is not None
# assert isinstance(results, list)
# assert len(results) == 1
# def test_calcs_user(self, client, example_elastic_calc, test_user_auth, no_warn):
# rv = client.get('/repo/?owner=user', headers=test_user_auth)
# assert rv.status_code == 200
# data = json.loads(rv.data)
# results = data.get('results', None)
# assert results is not None
# assert len(results) >= 1
# def test_calcs_user_authrequired(self, client, example_elastic_calc, no_warn):
# rv = client.get('/repo/?owner=user')
# assert rv.status_code == 401
# def test_calcs_user_invisible(self, client, example_elastic_calc, test_other_user_auth, no_warn):
# rv = client.get('/repo/?owner=user', headers=test_other_user_auth)
# assert rv.status_code == 200
# data = json.loads(rv.data)
# results = data.get('results', None)
# assert results is not None
# assert len(results) == 0
@pytest.mark.parametrize('calcs, owner, auth', [
(2, 'all', 'none'),
(2, 'all', 'test_user'),
(1, 'user', 'test_user'),
(2, 'user', 'other_test_user'),
(0, 'staging', 'test_user'),
(1, 'staging', 'other_test_user'),
])
def test_search(self, client, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
rv = client.get('/repo/?owner=%s' % owner, headers=auth)
assert rv.status_code == 200
data = json.loads(rv.data)
results = data.get('results', None)
assert results is not None
assert isinstance(results, list)
assert len(results) == calcs
if calcs > 0:
for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
assert key in results[0]
def test_calcs_pagination(self, client, example_elastic_calcs, no_warn):
rv = client.get('/repo/?page=1&per_page=1')
assert rv.status_code == 200
data = json.loads(rv.data)
results = data.get('results', None)
assert results is not None
assert isinstance(results, list)
assert len(results) == 1
def test_search_user_authrequired(self, client, example_elastic_calcs, no_warn):
rv = client.get('/repo/?owner=user')
assert rv.status_code == 401
class TestRaw(UploadFilesBasedTests):
......
......@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from elasticsearch_dsl import Q
from nomad import datamodel, search, processing, parsing
from nomad.search import Entry
......@@ -46,10 +48,15 @@ def test_index_upload(elastic, processed: processing.Upload):
def create_entry(calc_with_metadata: datamodel.CalcWithMetadata):
search.Entry.from_calc_with_metadata(calc_with_metadata).save()
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
assert_entry(calc_with_metadata.calc_id)
def assert_entry(calc_id):
calc = Entry.get(calc_id)
assert calc is not None
search = Entry.search().query(Q('term', calc_id=calc_id))[0:10]
assert search.count() == 1
results = list(hit.to_dict() for hit in search)
assert results[0]['calc_id'] == calc_id
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment