Refactored search->repo.

nomad/api.py

@@ -6,7 +6,7 @@ from elasticsearch.exceptions import NotFoundError
 from nomad import config, files
 from nomad.utils import get_logger, create_uuid
 from nomad.processing import Upload, Calc, NotAllowedDuringProcessing
-from nomad.search import CalcElasticDocument
+from nomad.repo import RepoCalc
 from nomad.user import me
 
 base_path = config.services.api_base_path
@@ -295,7 +295,7 @@ class RepoCalcRes(Resource):
         :returns: the repository calculation entry
         """
         try:
-            return CalcElasticDocument.get(id='%s/%s' % (upload_hash, calc_hash)).json_dict, 200
+            return RepoCalc.get(id='%s/%s' % (upload_hash, calc_hash)).json_dict, 200
         except NotFoundError:
             abort(404, message='There is no calculation for %s/%s' % (upload_hash, calc_hash))
         except Exception as e:
@@ -372,7 +372,7 @@ class RepoCalcsRes(Resource):
         assert per_page > 0
 
         try:
-            search = CalcElasticDocument.search().query('match_all')
+            search = RepoCalc.search().query('match_all')
             search = search[(page - 1) * per_page: page * per_page]
             return {
                 'pagination': {

nomad/processing/__init__.py

@@ -30,7 +30,7 @@ This module is structures into our *celery app* (``app.py``), the task definitio
 based on file storage notifications (``handler.py``, ``handlerdaemon.py``).
 
 This module does not contain the functions to do the actual work. Those are encapsulated
-in :py:mod:`nomad.files`, :py:mod:`nomad.search`, :py:mod:`nomad.users`,
+in :py:mod:`nomad.files`, :py:mod:`nomad.repo`, :py:mod:`nomad.users`,
 :py:mod:`nomad.parsing`, and :py:mod:`nomad.normalizing`.
 
 Processing app

nomad/processing/data.py

@@ -37,7 +37,7 @@ import mongoengine.errors
 import logging
 
 from nomad import config, files, utils
-from nomad.search import CalcElasticDocument
+from nomad.repo import RepoCalc
 from nomad.user import User, me
 from nomad.processing.base import Proc, process, task, PENDING
 from nomad.parsing import LocalBackend, parsers, parser_dict
@@ -96,7 +96,7 @@ class Calc(Proc):
             files.delete_archive(self.archive_id)
 
         # delete the search index entry
-        elastic_entry = CalcElasticDocument.get(self.archive_id)
+        elastic_entry = RepoCalc.get(self.archive_id)
         if elastic_entry is not None:
             elastic_entry.delete()
 
@@ -159,7 +159,7 @@ class Calc(Proc):
     def archiving(self):
         upload_hash, calc_hash = self.archive_id.split('/')
         # persist to elastic search
-        CalcElasticDocument.create_from_backend(
+        RepoCalc.create_from_backend(
             self._parser_backend,
             upload_hash=upload_hash,
             calc_hash=calc_hash,
@@ -314,7 +314,7 @@ class Upload(Proc):
             self.fail('could not create upload hash', e)
             return
 
-        if CalcElasticDocument.upload_exists(self.upload_hash):
+        if RepoCalc.upload_exists(self.upload_hash):
             self.fail('The same file was already uploaded and processed.', level=logging.INFO)
             return
 

nomad/search.py → nomad/repo.py

@@ -36,7 +36,7 @@ key_mappings = {
 class AlreadyExists(Exception): pass
 
 
-class CalcElasticDocument(ElasticDocument):
+class RepoCalc(ElasticDocument):
     """
     Elastic search document that represents a calculation. It is supposed to be a
     component of :class:`Calc`. Should only be created by its parent :class:`Calc`
@@ -72,7 +72,7 @@ class CalcElasticDocument(ElasticDocument):
     @classmethod
     def create_from_backend(
             cls, backend: LocalBackend, upload_id: str, upload_hash: str, calc_hash: str,
-            **kwargs) -> 'CalcElasticDocument':
+            **kwargs) -> 'RepoCalc':
         """
         Create a new calculation instance in elastic search. The data from the given backend
         will be used. Additional meta-data can be given as *kwargs*. ``upload_id``,
@@ -151,7 +151,7 @@ class CalcElasticDocument(ElasticDocument):
 
 if 'sphinx' not in sys.modules:
     try:
-        CalcElasticDocument.init()
+        RepoCalc.init()
     except elasticsearch.exceptions.RequestError as e:
         if e.status_code == 400 and 'resource_already_exists_exception' in e.error:
             pass  # happens if two services try this at the same time

tests/conftest.py

@@ -41,5 +41,5 @@ def mocksearch(monkeypatch):
     def upload_exists(upload_hash):
         return upload_hash in uploads
 
-    monkeypatch.setattr('nomad.search.CalcElasticDocument.create_from_backend', create_from_backend)
-    monkeypatch.setattr('nomad.search.CalcElasticDocument.upload_exists', upload_exists)
+    monkeypatch.setattr('nomad.repo.RepoCalc.create_from_backend', create_from_backend)
+    monkeypatch.setattr('nomad.repo.RepoCalc.upload_exists', upload_exists)

tests/processing/test_data.py

@@ -27,7 +27,7 @@ from nomad import config, files
 from nomad.processing import Upload, Calc
 from nomad.processing.base import task as task_decorator
 from nomad.user import me
-from nomad.search import CalcElasticDocument
+from nomad.repo import RepoCalc
 
 from tests.test_files import example_file, empty_file
 
@@ -90,7 +90,7 @@ def test_processing_doublets(uploaded_id, celery_session_worker, caplog):
 
     upload = run_processing(uploaded_id)
     assert upload.status == 'SUCCESS'
-    assert CalcElasticDocument.upload_exists(upload.upload_hash)  # pylint: disable=E1101
+    assert RepoCalc.upload_exists(upload.upload_hash)  # pylint: disable=E1101
 
     upload = run_processing(uploaded_id)
     assert upload.status == 'FAILURE'

tests/test_api.py

@@ -24,7 +24,7 @@ from tests.test_files import assert_exists  # noqa
 from tests.test_files import clear_files, archive_id  # noqa pylint: disable=unused-import
 from tests.test_normalizing import normalized_vasp_example  # noqa pylint: disable=unused-import
 from tests.test_parsing import parsed_vasp_example  # noqa pylint: disable=unused-import
-from tests.test_search import example_elastic_calc  # noqa pylint: disable=unused-import
+from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
 
 
 @pytest.fixture(scope='function')

tests/test_search.py → tests/test_repo.py

@@ -21,7 +21,7 @@ from elasticsearch import NotFoundError
 
 from nomad import config
 from nomad.parsing import LocalBackend
-from nomad.search import AlreadyExists, CalcElasticDocument, key_mappings
+from nomad.repo import AlreadyExists, RepoCalc, key_mappings
 
 from tests.test_normalizing import normalized_vasp_example  # pylint: disable=unused-import
 from tests.test_parsing import parsed_vasp_example  # pylint: disable=unused-import
@@ -30,16 +30,16 @@ from tests.test_files import assert_not_exists
 
 @pytest.fixture(scope='function')
 def example_elastic_calc(normalized_vasp_example: LocalBackend, caplog) \
-        -> Generator[CalcElasticDocument, None, None]:
+        -> Generator[RepoCalc, None, None]:
     try:
         caplog.set_level(logging.ERROR)
-        CalcElasticDocument.get(id='test_upload_hash/test_calc_hash').delete()
+        RepoCalc.get(id='test_upload_hash/test_calc_hash').delete()
     except Exception:
         pass
     finally:
         caplog.set_level(logging.WARNING)
 
-    entry = CalcElasticDocument.create_from_backend(
+    entry = RepoCalc.create_from_backend(
         normalized_vasp_example,
         upload_hash='test_upload_hash',
         calc_hash='test_calc_hash',
@@ -59,27 +59,27 @@ def example_elastic_calc(normalized_vasp_example: LocalBackend, caplog) \
         caplog.set_level(logging.WARNING)
 
 
-def assert_elastic_calc(calc: CalcElasticDocument):
+def assert_elastic_calc(calc: RepoCalc):
     assert calc is not None
-    for property in CalcElasticDocument._doc_type.mapping:
+    for property in RepoCalc._doc_type.mapping:
         property = key_mappings.get(property, property)
         assert getattr(calc, property) is not None
 
 
-def test_create_elasitc_calc(example_elastic_calc: CalcElasticDocument):
+def test_create_elasitc_calc(example_elastic_calc: RepoCalc):
     assert_elastic_calc(example_elastic_calc)
-    assert CalcElasticDocument.upload_exists(example_elastic_calc.upload_hash)
+    assert RepoCalc.upload_exists(example_elastic_calc.upload_hash)
 
-    get_result: CalcElasticDocument = CalcElasticDocument.get(
+    get_result: RepoCalc = RepoCalc.get(
         id='%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
     assert_elastic_calc(get_result)
 
 
 def test_create_existing_elastic_calc(
-        example_elastic_calc: CalcElasticDocument, normalized_vasp_example, caplog):
+        example_elastic_calc: RepoCalc, normalized_vasp_example, caplog):
     try:
         caplog.set_level(logging.ERROR)
-        CalcElasticDocument.create_from_backend(
+        RepoCalc.create_from_backend(
             normalized_vasp_example,
             upload_hash='test_upload_hash',
             calc_hash='test_calc_hash',
@@ -94,13 +94,13 @@ def test_create_existing_elastic_calc(
         assert False
 
 
-def test_delete_elastic_calc(example_elastic_calc: CalcElasticDocument, caplog):
+def test_delete_elastic_calc(example_elastic_calc: RepoCalc, caplog):
     example_elastic_calc.delete()
 
     assert_not_exists(config.files.archive_bucket, 'test_upload_hash/test_calc_hash')
     try:
         caplog.set_level(logging.ERROR)
-        CalcElasticDocument.get(id='test_upload_hash/test_calc_hash')
+        RepoCalc.get(id='test_upload_hash/test_calc_hash')
         assert False
     except NotFoundError:
         pass