Commit 7e1ab14a authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added first search tests.

parent e14802ae
......@@ -149,13 +149,14 @@ class Upload(Base, datamodel.Upload): # type: ignore
repo_db.add(coe_upload)
# add calculations and metadata
has_calcs = False
calcs = []
for calc in upload.calcs:
has_calcs = True
coe_upload._add_calculation(calc.to(CalcWithMetadata), upload_metadata.get(calc.mainfile))
calcs.append(
coe_upload._add_calculation(
calc.to(CalcWithMetadata), upload_metadata.get(calc.mainfile)))
# commit
if has_calcs:
if len(calcs) > 0:
# empty upload case
repo_db.commit()
result = coe_upload.coe_upload_id
......@@ -171,7 +172,7 @@ class Upload(Base, datamodel.Upload): # type: ignore
return result
def _add_calculation(self, calc: CalcWithMetadata, calc_metadata: dict) -> None:
def _add_calculation(self, calc: CalcWithMetadata, calc_metadata: dict) -> Calc:
repo_db = infrastructure.repository_db
# table based properties
......@@ -266,6 +267,8 @@ class Upload(Base, datamodel.Upload): # type: ignore
for reference in calc_metadata.get('references', []):
self._add_citation(coe_calc, reference, 'EXTERNAL')
return coe_calc
def _add_citation(self, coe_calc: Calc, value: str, kind: str) -> None:
repo_db = infrastructure.repository_db
citation = repo_db.query(Citation).filter_by(value=value, kind=kind).first()
......
......@@ -39,7 +39,7 @@ FSConfig = namedtuple('FSConfig', ['tmp', 'objects'])
RepositoryDBConfig = namedtuple('RepositoryDBConfig', ['host', 'port', 'dbname', 'user', 'password'])
""" Used to configure access to NOMAD-coe repository db. """
ElasticConfig = namedtuple('ElasticConfig', ['host', 'port', 'index_name', 'coe_repo_calcs_index_name'])
ElasticConfig = namedtuple('ElasticConfig', ['host', 'port', 'index_name'])
""" Used to configure elastic search. """
MongoConfig = namedtuple('MongoConfig', ['host', 'port', 'db_name'])
......@@ -91,20 +91,19 @@ fs = FSConfig(
elastic = ElasticConfig(
host=os.environ.get('NOMAD_ELASTIC_HOST', 'localhost'),
port=int(os.environ.get('NOMAD_ELASTIC_PORT', 9200)),
index_name=os.environ.get('NOMAD_ELASTIC_INDEX_NAME', 'calcs'),
coe_repo_calcs_index_name='coe_repo_calcs'
index_name=os.environ.get('NOMAD_ELASTIC_INDEX_NAME', 'nomad_fairdi_calcs')
)
repository_db = RepositoryDBConfig(
host=os.environ.get('NOMAD_COE_REPO_DB_HOST', 'localhost'),
port=int(os.environ.get('NOMAD_COE_REPO_DB_PORT', 5432)),
dbname=os.environ.get('NOMAD_COE_REPO_DB_NAME', 'nomad'),
dbname=os.environ.get('NOMAD_COE_REPO_DB_NAME', 'nomad_fairdi_repo_db'),
user=os.environ.get('NOMAD_COE_REPO_DB_USER', 'postgres'),
password=os.environ.get('NOMAD_COE_REPO_PASSWORD', 'nomad')
)
mongo = MongoConfig(
host=os.environ.get('NOMAD_MONGO_HOST', 'localhost'),
port=int(os.environ.get('NOMAD_MONGO_PORT', 27017)),
db_name=os.environ.get('NOMAD_MONGO_DB_NAME', 'users')
db_name=os.environ.get('NOMAD_MONGO_DB_NAME', 'nomad_fairdi')
)
logstash = LogstashConfig(
enabled=True,
......
......@@ -61,19 +61,19 @@ class Entity():
A common base class for all nomad entities. It provides the functions necessary
to apply transformations.
"""
mappings: Dict[Type['Entity'], Callable[['Entity'], 'Entity']] = dict()
mappings: Dict = dict()
@classmethod
def register_mapping(
cls, from_type: Type['Entity'], mapping: Callable[['Entity'], 'Entity']):
cls, from_type: Type[T], mapping: Callable[[T], 'Entity']):
"""
Register a mapping from instances of another calc representation to instances of
:class:`CalcWithMetadata`.
this calc representation.
Arguments:
from_type: The source calc type of the mapping.
mapping: The mapping itself as a callable that takes a source object of the
source calc type and returns an instance of :class:`CalcWithMetadata`.
source calc type and returns an instance of target calc type.
"""
cls.mappings[from_type] = mapping
......
......@@ -89,7 +89,8 @@ def setup_elastic():
try:
from nomad.search import Entry
Entry.init()
Entry.init(index=config.elastic.index_name)
logger.info('initialized elastic index', index_name=config.elastic.index_name)
except RequestError as e:
if e.status_code == 400 and 'resource_already_exists_exception' in e.error:
pass # happens if two services try this at the same time
......@@ -202,10 +203,10 @@ def reset():
try:
if not elastic_client:
setup_elastic()
elastic_client.indices.delete(index=config.elastic.index_name)
from nomad.search import Entry
Entry.init()
logger.info('elastic index resetted')
elastic_client.indices.delete(index=config.elastic.index_name)
from nomad.search import Entry
Entry.init(index=config.elastic.index_name)
logger.info('elastic index resetted')
except Exception as e:
logger.error('exception resetting elastic', exc_info=e)
......
......@@ -109,7 +109,7 @@ class Calc(InnerDoc):
class Entry(Document, datamodel.Entity):
class Index:
name = config.elastic.coe_repo_calcs_index_name
name = config.elastic.index_name
calc_id = Keyword()
upload_id = Keyword()
......@@ -120,6 +120,11 @@ class Entry(Document, datamodel.Entity):
self.calc_id = calc_id
self.upload_id = upload_id
@classmethod
def from_calc_with_metadata(cls, source: datamodel.CalcWithMetadata) -> 'Entry':
target = Entry(source.upload_id, source.calc_id)
return target
def persist(self, **kwargs):
"""
Persist this entry to elastic search. Kwargs are passed to elastic search.
......@@ -183,3 +188,6 @@ class Entry(Document, datamodel.Entity):
data['upload_time'] = data['upload_time'].isoformat()
return {key: value for key, value in data.items() if value is not None}
Entry.register_mapping(datamodel.CalcWithMetadata, Entry.from_calc_with_metadata)
......@@ -10,6 +10,7 @@ from threading import Lock, Thread
import asyncore
import time
import pytest
import elasticsearch.exceptions
from nomad import config, infrastructure
......@@ -109,9 +110,17 @@ def mockmongo(monkeypatch):
connection.drop_database('test_db')
@pytest.fixture(scope='session')
def elastic():
@pytest.fixture(scope='function')
def elastic(monkeysession):
monkeysession.setattr('nomad.config.elastic', config.elastic._replace(index_name='test_nomad_fairdi_calcs'))
infrastructure.setup_elastic()
try:
from nomad.search import Entry
Entry._index.delete()
Entry.init(index=config.elastic.index_name)
except elasticsearch.exceptions.NotFoundError:
pass
assert infrastructure.elastic_client is not None
......@@ -121,7 +130,7 @@ def create_repository_db(monkeysession=None, **kwargs):
A generator that sets up and tears down a test db and monkeypatches it to the
respective global infrastructure variables.
"""
db_args = dict(dbname='test_nomad_fair_repo_db')
db_args = dict(dbname='test_nomad_fairdi_repo_db')
db_args.update(**kwargs)
old_config = config.repository_db
......@@ -174,7 +183,7 @@ def repository_db(monkeysession):
@pytest.fixture(scope='function')
def expandable_repo_db(monkeysession, repository_db):
with create_repository_db(monkeysession, dbname='test_nomad_fair_expandable_repo_db', exists=False) as db:
with create_repository_db(monkeysession, dbname='test_nomad_fairdi_expandable_repo_db', exists=False) as db:
yield db
......@@ -256,6 +265,7 @@ class SMTPServerThread(Thread):
super().__init__()
self.messages = messages
self.host_port = None
self.smtp = None
def run(self):
_messages = self.messages
......@@ -273,7 +283,8 @@ class SMTPServerThread(Thread):
pass
def close(self):
self.smtp.close()
if self.smtp is not None:
self.smtp.close()
class SMTPServerFixture:
......
......@@ -12,8 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import nomad.search # pylint: disable=W0611
from nomad import datamodel, search
def test_helloworld(elastic):
def test_init_mapping(elastic):
pass
def test_index_calc(elastic):
calc = datamodel.CalcWithMetadata(upload_id='test_upload', calc_id='test_calc')
calc.to(search.Entry).save(op_type='create')
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment