Commit 7c829119 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored search->repo.

parent 9be44de2
......@@ -6,7 +6,7 @@ from elasticsearch.exceptions import NotFoundError
from nomad import config, files
from nomad.utils import get_logger, create_uuid
from nomad.processing import Upload, Calc, NotAllowedDuringProcessing
from nomad.search import CalcElasticDocument
from nomad.repo import RepoCalc
from nomad.user import me
base_path = config.services.api_base_path
......@@ -295,7 +295,7 @@ class RepoCalcRes(Resource):
:returns: the repository calculation entry
"""
try:
return CalcElasticDocument.get(id='%s/%s' % (upload_hash, calc_hash)).json_dict, 200
return RepoCalc.get(id='%s/%s' % (upload_hash, calc_hash)).json_dict, 200
except NotFoundError:
abort(404, message='There is no calculation for %s/%s' % (upload_hash, calc_hash))
except Exception as e:
......@@ -372,7 +372,7 @@ class RepoCalcsRes(Resource):
assert per_page > 0
try:
search = CalcElasticDocument.search().query('match_all')
search = RepoCalc.search().query('match_all')
search = search[(page - 1) * per_page: page * per_page]
return {
'pagination': {
......
......@@ -30,7 +30,7 @@ This module is structures into our *celery app* (``app.py``), the task definitio
based on file storage notifications (``handler.py``, ``handlerdaemon.py``).
This module does not contain the functions to do the actual work. Those are encapsulated
in :py:mod:`nomad.files`, :py:mod:`nomad.search`, :py:mod:`nomad.users`,
in :py:mod:`nomad.files`, :py:mod:`nomad.repo`, :py:mod:`nomad.users`,
:py:mod:`nomad.parsing`, and :py:mod:`nomad.normalizing`.
Processing app
......
......@@ -37,7 +37,7 @@ import mongoengine.errors
import logging
from nomad import config, files, utils
from nomad.search import CalcElasticDocument
from nomad.repo import RepoCalc
from nomad.user import User, me
from nomad.processing.base import Proc, process, task, PENDING
from nomad.parsing import LocalBackend, parsers, parser_dict
......@@ -96,7 +96,7 @@ class Calc(Proc):
files.delete_archive(self.archive_id)
# delete the search index entry
elastic_entry = CalcElasticDocument.get(self.archive_id)
elastic_entry = RepoCalc.get(self.archive_id)
if elastic_entry is not None:
elastic_entry.delete()
......@@ -159,7 +159,7 @@ class Calc(Proc):
def archiving(self):
upload_hash, calc_hash = self.archive_id.split('/')
# persist to elastic search
CalcElasticDocument.create_from_backend(
RepoCalc.create_from_backend(
self._parser_backend,
upload_hash=upload_hash,
calc_hash=calc_hash,
......@@ -314,7 +314,7 @@ class Upload(Proc):
self.fail('could not create upload hash', e)
return
if CalcElasticDocument.upload_exists(self.upload_hash):
if RepoCalc.upload_exists(self.upload_hash):
self.fail('The same file was already uploaded and processed.', level=logging.INFO)
return
......
......@@ -36,7 +36,7 @@ key_mappings = {
class AlreadyExists(Exception): pass
class CalcElasticDocument(ElasticDocument):
class RepoCalc(ElasticDocument):
"""
Elastic search document that represents a calculation. It is supposed to be a
component of :class:`Calc`. Should only be created by its parent :class:`Calc`
......@@ -72,7 +72,7 @@ class CalcElasticDocument(ElasticDocument):
@classmethod
def create_from_backend(
cls, backend: LocalBackend, upload_id: str, upload_hash: str, calc_hash: str,
**kwargs) -> 'CalcElasticDocument':
**kwargs) -> 'RepoCalc':
"""
Create a new calculation instance in elastic search. The data from the given backend
will be used. Additional meta-data can be given as *kwargs*. ``upload_id``,
......@@ -151,7 +151,7 @@ class CalcElasticDocument(ElasticDocument):
if 'sphinx' not in sys.modules:
try:
CalcElasticDocument.init()
RepoCalc.init()
except elasticsearch.exceptions.RequestError as e:
if e.status_code == 400 and 'resource_already_exists_exception' in e.error:
pass # happens if two services try this at the same time
......
......@@ -41,5 +41,5 @@ def mocksearch(monkeypatch):
def upload_exists(upload_hash):
return upload_hash in uploads
monkeypatch.setattr('nomad.search.CalcElasticDocument.create_from_backend', create_from_backend)
monkeypatch.setattr('nomad.search.CalcElasticDocument.upload_exists', upload_exists)
monkeypatch.setattr('nomad.repo.RepoCalc.create_from_backend', create_from_backend)
monkeypatch.setattr('nomad.repo.RepoCalc.upload_exists', upload_exists)
......@@ -27,7 +27,7 @@ from nomad import config, files
from nomad.processing import Upload, Calc
from nomad.processing.base import task as task_decorator
from nomad.user import me
from nomad.search import CalcElasticDocument
from nomad.repo import RepoCalc
from tests.test_files import example_file, empty_file
......@@ -90,7 +90,7 @@ def test_processing_doublets(uploaded_id, celery_session_worker, caplog):
upload = run_processing(uploaded_id)
assert upload.status == 'SUCCESS'
assert CalcElasticDocument.upload_exists(upload.upload_hash) # pylint: disable=E1101
assert RepoCalc.upload_exists(upload.upload_hash) # pylint: disable=E1101
upload = run_processing(uploaded_id)
assert upload.status == 'FAILURE'
......
......@@ -24,7 +24,7 @@ from tests.test_files import assert_exists # noqa
from tests.test_files import clear_files, archive_id # noqa pylint: disable=unused-import
from tests.test_normalizing import normalized_vasp_example # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_vasp_example # noqa pylint: disable=unused-import
from tests.test_search import example_elastic_calc # noqa pylint: disable=unused-import
from tests.test_repo import example_elastic_calc # noqa pylint: disable=unused-import
@pytest.fixture(scope='function')
......
......@@ -21,7 +21,7 @@ from elasticsearch import NotFoundError
from nomad import config
from nomad.parsing import LocalBackend
from nomad.search import AlreadyExists, CalcElasticDocument, key_mappings
from nomad.repo import AlreadyExists, RepoCalc, key_mappings
from tests.test_normalizing import normalized_vasp_example # pylint: disable=unused-import
from tests.test_parsing import parsed_vasp_example # pylint: disable=unused-import
......@@ -30,16 +30,16 @@ from tests.test_files import assert_not_exists
@pytest.fixture(scope='function')
def example_elastic_calc(normalized_vasp_example: LocalBackend, caplog) \
-> Generator[CalcElasticDocument, None, None]:
-> Generator[RepoCalc, None, None]:
try:
caplog.set_level(logging.ERROR)
CalcElasticDocument.get(id='test_upload_hash/test_calc_hash').delete()
RepoCalc.get(id='test_upload_hash/test_calc_hash').delete()
except Exception:
pass
finally:
caplog.set_level(logging.WARNING)
entry = CalcElasticDocument.create_from_backend(
entry = RepoCalc.create_from_backend(
normalized_vasp_example,
upload_hash='test_upload_hash',
calc_hash='test_calc_hash',
......@@ -59,27 +59,27 @@ def example_elastic_calc(normalized_vasp_example: LocalBackend, caplog) \
caplog.set_level(logging.WARNING)
def assert_elastic_calc(calc: CalcElasticDocument):
def assert_elastic_calc(calc: RepoCalc):
assert calc is not None
for property in CalcElasticDocument._doc_type.mapping:
for property in RepoCalc._doc_type.mapping:
property = key_mappings.get(property, property)
assert getattr(calc, property) is not None
def test_create_elasitc_calc(example_elastic_calc: CalcElasticDocument):
def test_create_elasitc_calc(example_elastic_calc: RepoCalc):
assert_elastic_calc(example_elastic_calc)
assert CalcElasticDocument.upload_exists(example_elastic_calc.upload_hash)
assert RepoCalc.upload_exists(example_elastic_calc.upload_hash)
get_result: CalcElasticDocument = CalcElasticDocument.get(
get_result: RepoCalc = RepoCalc.get(
id='%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
assert_elastic_calc(get_result)
def test_create_existing_elastic_calc(
example_elastic_calc: CalcElasticDocument, normalized_vasp_example, caplog):
example_elastic_calc: RepoCalc, normalized_vasp_example, caplog):
try:
caplog.set_level(logging.ERROR)
CalcElasticDocument.create_from_backend(
RepoCalc.create_from_backend(
normalized_vasp_example,
upload_hash='test_upload_hash',
calc_hash='test_calc_hash',
......@@ -94,13 +94,13 @@ def test_create_existing_elastic_calc(
assert False
def test_delete_elastic_calc(example_elastic_calc: CalcElasticDocument, caplog):
def test_delete_elastic_calc(example_elastic_calc: RepoCalc, caplog):
example_elastic_calc.delete()
assert_not_exists(config.files.archive_bucket, 'test_upload_hash/test_calc_hash')
try:
caplog.set_level(logging.ERROR)
CalcElasticDocument.get(id='test_upload_hash/test_calc_hash')
RepoCalc.get(id='test_upload_hash/test_calc_hash')
assert False
except NotFoundError:
pass
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment