Merge branch 'v0.9.0' into 'master'

V0.9.0

See merge request !188

tests/metainfo/test_metainfo.py

@@ -20,7 +20,7 @@ import datetime
 from nomad.metainfo.metainfo import (
     MSection, MCategory, Section, Quantity, SubSection, Definition, Package, DeriveError,
     MetainfoError, Environment, MResource, Datetime, Annotation, SectionAnnotation,
-    DefinitionAnnotation, Reference, MProxy, derived, SectionProxy)
+    DefinitionAnnotation, Reference, MProxy, derived, SectionProxy, JSON)
 from nomad.metainfo.example import Run, VaspRun, System, SystemHash, Parsing, SCC, m_package as example_package
 from nomad import utils
 from nomad.units import ureg
@@ -688,6 +688,22 @@ class TestDatatypes:
         assert obj.datetime is None
         assert obj.m_to_dict()['datetime'] is None
 
+    def test_json(self):
+        class TestSection(MSection):
+            json = Quantity(type=JSON)
+
+        obj = TestSection()
+        assert obj.json is None
+        assert 'json' not in obj.m_to_dict()
+
+        obj.json = dict(test_key='test_value')
+        assert obj.json is not None
+        assert isinstance(obj.m_to_dict()['json'], dict)
+
+        obj.json = None
+        assert obj.json is None
+        assert obj.m_to_dict()['json'] is None
+
 
 class TestEnvironment:
 

tests/metainfo/test_mongodb_extension.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import numpy as np
+from nomad.metainfo import MSection, Section, Quantity, SubSection
+from nomad.metainfo.mongoengine_extension import MongoDocument, Mongo
+
+
+class B(MSection):
+    m_def = Section(a_mongo=MongoDocument())
+    value = Quantity(type=str, a_mongo=Mongo())
+
+
+class C(MSection):
+    m_def = Section(a_mongo=MongoDocument())
+    value = Quantity(type=str, a_mongo=Mongo())
+
+
+class D(MSection):
+    m_def = Section()
+    value = Quantity(type=str, a_mongo=Mongo())
+
+
+class A(MSection):
+    """Root level document with primary key.
+    """
+    m_def = Section(a_mongo=MongoDocument())
+    primary_id = Quantity(type=str, a_mongo=Mongo(primary_key=True))
+    array = Quantity(type=float, shape=[2], a_mongo=Mongo())
+    not_in_mongo = Quantity(type=str)
+    value1 = Quantity(type=int, a_mongo=Mongo())
+    value2 = Quantity(type=int, a_mongo=Mongo())
+    b = SubSection(sub_section=B.m_def)
+    c = SubSection(sub_section=C.m_def, repeats=True)
+    d = SubSection(sub_section=D.m_def)
+
+
+def test_create_new(mongo):
+    a = A()
+    a.primary_id = "123"
+    a.not_in_mongo = "not_in_mongo"
+    b = a.m_create(B)
+    b.value = "b_value"
+    c = a.m_create(C)
+    c.value = "c_value"
+    c = a.m_create(C)
+    c.value = "c_value"
+
+    # Create JSON with the values that are supposed to be in mongo
+    a_dict = a.m_to_dict()
+    del a_dict["not_in_mongo"]
+    a_json = json.dumps(a_dict, sort_keys=True)
+
+    # Store to mongo
+    mongo_doc = a.a_mongo
+    mongo_doc.save()
+
+    # Retrieve from mongo, and convert to JSON
+    a_from_db = A.m_def.a_mongo.get(primary_id="123")
+    a_from_db_json = json.dumps(a_from_db.m_to_dict(), sort_keys=True)
+
+    # Test equality of the JSON serializations
+    assert a_json == a_from_db_json
+
+
+def test_update_with_new():
+    a = A()
+    a.primary_id = "123"
+    a.value1 = 1
+    a.value2 = 2
+
+    # Store to mongo
+    a.a_mongo.save()
+
+    # Update with new document that has the same ID
+    a_new = A()
+    a_new.primary_id = "123"
+    a_new.value2 = 3
+    a_new.a_mongo.save()
+
+    # Check that the document has only partly been updated
+    a_from_db = A.m_def.a_mongo.get(primary_id="123")
+    assert a_from_db.value1 == 1
+    assert a_from_db.value2 == 3
+
+
+def test_update_self():
+    a = A()
+    a.primary_id = "123"
+    a.value1 = 1
+    a.value2 = 2
+
+    # Store to mongo
+    a.a_mongo.save()
+
+    # Update the metainfo and resave
+    a.value2 = 3
+    a.a_mongo.save()
+
+    # Check that the document has only partly been updated
+    a_from_db = A.m_def.a_mongo.get(primary_id="123")
+    assert a_from_db.value1 == 1
+    assert a_from_db.value2 == 3
+
+
+def test_annotations(mongo):
+    """Test that non-annotated quantities and sections are not stored.
+    """
+    a = A()
+    a.primary_id = "123"
+    a.not_in_mongo = "not_in_mongo"
+    d = a.m_create(D)
+    d.value = "b_value"
+
+    # Store to mongo
+    a.a_mongo.save()
+
+    # Check that values do not exist in mongodb
+    a_from_db = A.m_def.a_mongo.get(primary_id="123")
+    assert a_from_db.not_in_mongo is None
+    assert a_from_db.d is None
+
+
+def test_repeated_subsections():
+    a = A()
+    a.primary_id = "123"
+    c = a.m_create(C)
+    c.value = "c_value"
+    c = a.m_create(C)
+    c.value = "c_value"
+
+    # Store to mongo
+    a.a_mongo.save()
+
+    # Check that both sections are stored in mongodb
+    a_from_db = A.m_def.a_mongo.get(primary_id="123")
+    assert len(a_from_db.c) == 2
+
+
+def test_arrays():
+    a = A()
+    a.primary_id = "123"
+    a.array = np.array([1.2, 3.4])
+
+    # Store to mongo
+    a.a_mongo.save()
+
+    # Check that array was correctly save
+    a_from_db = A.m_def.a_mongo.get(primary_id="123")
+    assert a_from_db.array == [1.2, 3.4]

tests/normalizing/conftest.py

@@ -16,8 +16,9 @@ import pytest
 
 from ase import Atoms
 
-from nomad.parsing.legacy import Backend
 from nomad.normalizing import normalizers
+from nomad.datamodel import EntryArchive
+from nomad.datamodel.metainfo.public import section_system as System
 
 from tests.test_parsing import parsed_vasp_example  # pylint: disable=unused-import
 from tests.test_parsing import parsed_template_example  # pylint: disable=unused-import
@@ -26,289 +27,260 @@ from tests.test_parsing import parse_file
 from tests.test_parsing import parsed_template_no_system  # pylint: disable=unused-import
 
 
-def run_normalize(backend: Backend) -> Backend:
-    status, _ = backend.status
-
-    assert status == 'ParseSuccess'
-
+def run_normalize(entry_archive: EntryArchive) -> EntryArchive:
     for normalizer_class in normalizers:
-        normalizer = normalizer_class(backend.entry_archive)
+        normalizer = normalizer_class(entry_archive)
         normalizer.normalize()
-    return backend
+    return entry_archive
 
 
 @pytest.fixture
-def normalized_vasp_example(parsed_vasp_example: Backend) -> Backend:
+def normalized_vasp_example(parsed_vasp_example: EntryArchive) -> EntryArchive:
     return run_normalize(parsed_vasp_example)
 
 
 @pytest.fixture
-def normalized_example(parsed_example: Backend) -> Backend:
+def normalized_example(parsed_example: EntryArchive) -> EntryArchive:
     return run_normalize(parsed_example)
 
 
 @pytest.fixture
-def normalized_template_example(parsed_template_example) -> Backend:
+def normalized_template_example(parsed_template_example) -> EntryArchive:
     return run_normalize(parsed_template_example)
 
 
-def run_normalize_for_structure(atoms: Atoms) -> Backend:
+def run_normalize_for_structure(atoms: Atoms) -> EntryArchive:
     template = parsed_template_no_system()
 
     # Fill structural information
-    gid = template.openSection("section_system")
-    template.addArrayValues("atom_positions", atoms.get_positions() * 1E-10)
-    template.addArrayValues("atom_labels", atoms.get_chemical_symbols())
-    template.addArrayValues("simulation_cell", atoms.get_cell() * 1E-10)
-    template.addArrayValues("configuration_periodic_dimensions", atoms.get_pbc())
-    template.closeSection("section_system", gid)
+    system = template.section_run[0].m_create(System)
+    system.atom_positions = atoms.get_positions() * 1E-10
+    system.atom_labels = atoms.get_chemical_symbols()
+    system.simulation_cell = atoms.get_cell() * 1E-10
+    system.configuration_periodic_dimensions = atoms.get_pbc()
 
     return run_normalize(template)
 
 
 @pytest.fixture(scope='session')
-def single_point(two_d) -> Backend:
+def single_point(two_d) -> EntryArchive:
     return two_d
 
 
 @pytest.fixture(scope='session')
-def gw(two_d) -> Backend:
+def gw(two_d) -> EntryArchive:
     parser_name = "parsers/template"
     filepath = "tests/data/normalizers/gw.json"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def geometry_optimization() -> Backend:
+def geometry_optimization() -> EntryArchive:
     parser_name = "parsers/template"
     filepath = "tests/data/normalizers/fcc_crystal_structure.json"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def molecular_dynamics(bulk) -> Backend:
+def molecular_dynamics(bulk) -> EntryArchive:
     return bulk
 
 
 @pytest.fixture(scope='session')
-def phonon() -> Backend:
+def phonon() -> EntryArchive:
     parser_name = "parsers/phonopy"
     filepath = "tests/data/parsers/phonopy/phonopy-FHI-aims-displacement-01/control.in"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def elastic() -> Backend:
+def elastic() -> EntryArchive:
     parser_name = "parsers/elastic"
     filepath = "tests/data/parsers/elastic/diamond/INFO_ElaStic"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def bulk() -> Backend:
+def bulk() -> EntryArchive:
     parser_name = "parsers/cp2k"
     filepath = "tests/data/normalizers/cp2k_bulk_md/si_md.out"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def two_d() -> Backend:
+def two_d() -> EntryArchive:
     parser_name = "parsers/fhi-aims"
     filepath = "tests/data/normalizers/fhiaims_2d_singlepoint/aims.out"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def surface() -> Backend:
+def surface() -> EntryArchive:
     parser_name = "parsers/fhi-aims"
     filepath = "tests/data/normalizers/fhiaims_surface_singlepoint/PBE-light+tight-rho2.out"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def molecule() -> Backend:
+def molecule() -> EntryArchive:
     parser_name = "parsers/fhi-aims"
     filepath = "tests/data/normalizers/fhiaims_molecule_singlepoint/aims.out"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def atom() -> Backend:
+def atom() -> EntryArchive:
     parser_name = "parsers/gaussian"
     filepath = "tests/data/normalizers/gaussian_atom_singlepoint/m9b7.out"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def one_d() -> Backend:
+def one_d() -> EntryArchive:
     parser_name = "parsers/exciting"
     filepath = "tests/data/normalizers/exciting_1d_singlepoint/INFO.OUT"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def bands_unpolarized_gap_indirect() -> Backend:
+def bands_unpolarized_gap_indirect() -> EntryArchive:
     parser_name = "parsers/vasp"
     filepath = "tests/data/normalizers/band_structure/unpolarized_gap/vasprun.xml.bands.xz"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def bands_polarized_no_gap() -> Backend:
+def bands_polarized_no_gap() -> EntryArchive:
     parser_name = "parsers/vasp"
     filepath = "tests/data/normalizers/band_structure/polarized_no_gap/vasprun.xml.bands.xz"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def bands_unpolarized_no_gap() -> Backend:
+def bands_unpolarized_no_gap() -> EntryArchive:
     parser_name = "parsers/vasp"
     filepath = "tests/data/normalizers/band_structure/unpolarized_no_gap/vasprun.xml.bands.xz"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def bands_polarized_gap_indirect() -> Backend:
+def bands_polarized_gap_indirect() -> EntryArchive:
     parser_name = "parsers/vasp"
     filepath = "tests/data/normalizers/band_structure/polarized_gap/vasprun.xml.bands.xz"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def dos_si_vasp() -> Backend:
+def dos_si_vasp() -> EntryArchive:
     parser_name = "parsers/vasp"
     filepath = "tests/data/normalizers/dos/dos_si_vasp/vasprun.xml.relax2.xz"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def dos_si_exciting() -> Backend:
+def dos_si_exciting() -> EntryArchive:
     parser_name = "parsers/exciting"
     filepath = "tests/data/normalizers/dos/dos_si_exciting/INFO.OUT"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def dos_si_fhiaims() -> Backend:
+def dos_si_fhiaims() -> EntryArchive:
     parser_name = "parsers/fhi-aims"
     filepath = "tests/data/normalizers/dos/dos_si_fhiaims/aims.log"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def dos_polarized_vasp() -> Backend:
+def dos_polarized_vasp() -> EntryArchive:
     parser_name = "parsers/vasp"
     filepath = "tests/data/normalizers/dos/polarized_vasp/vasprun.xml.relax2.xz"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def dos_unpolarized_vasp() -> Backend:
+def dos_unpolarized_vasp() -> EntryArchive:
     parser_name = "parsers/vasp"
     filepath = "tests/data/normalizers/dos/unpolarized_vasp/vasprun.xml.xz"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def hash_exciting() -> Backend:
+def hash_exciting() -> EntryArchive:
     parser_name = "parsers/exciting"
     filepath = "tests/data/normalizers/hashes/exciting/INFO.OUT"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def hash_vasp(bands_unpolarized_gap_indirect) -> Backend:
+def hash_vasp(bands_unpolarized_gap_indirect) -> EntryArchive:
     return bands_unpolarized_gap_indirect
 
 
 @pytest.fixture(scope='session')
-def band_path_cF(bands_unpolarized_gap_indirect) -> Backend:
+def band_path_cF(bands_unpolarized_gap_indirect) -> EntryArchive:
     """Band structure calculation for a cP Bravais lattice.
     """
     return bands_unpolarized_gap_indirect
 
 
 @pytest.fixture(scope='session')
-def band_path_tP() -> Backend:
+def band_path_tP() -> EntryArchive:
     """Band structure calculation for a tP Bravais lattice.
     """
     parser_name = "parsers/vasp"
     filepath = "tests/data/normalizers/band_structure/tP/vasprun.xml.bands.xz"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def band_path_hP() -> Backend:
+def band_path_hP() -> EntryArchive:
     """Band structure calculation for a hP Bravais lattice.
     """
     parser_name = "parsers/vasp"
     filepath = "tests/data/normalizers/band_structure/hP/vasprun.xml.bands.xz"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def band_path_mP_nonstandard() -> Backend:
+def band_path_mP_nonstandard() -> EntryArchive:
     """Band structure calculation for a mP Bravais lattice with a non-standard
     lattice ordering.
     """
     parser_name = "parsers/vasp"
     filepath = "tests/data/normalizers/band_structure/mP_nonstandard/vasprun.xml.bands.xz"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)
 
 
 @pytest.fixture(scope='session')
-def band_path_cF_nonstandard() -> Backend:
+def band_path_cF_nonstandard() -> EntryArchive:
     """Band structure calculation for a mP Bravais lattice with a non-standard
     lattice ordering.
     """
     parser_name = "parsers/exciting"
     filepath = "tests/data/normalizers/band_structure/cF_nonstandard/INFO.OUT"
-    backend = parse_file((parser_name, filepath))
-    backend = run_normalize(backend)
-    return backend
+    archive = parse_file((parser_name, filepath))
+    return run_normalize(archive)

tests/normalizing/test_band_structure.py

@@ -40,20 +40,20 @@ def test_band_gaps(bands_unpolarized_no_gap, bands_polarized_no_gap, bands_unpol
         assert bs.reciprocal_cell.shape == (3, 3)
 
     # Unpolarized, no gaps
-    bs = bands_unpolarized_no_gap.entry_archive.section_run[0].section_single_configuration_calculation[0].section_k_band[0]
+    bs = bands_unpolarized_no_gap.section_run[0].section_single_configuration_calculation[0].section_k_band[0]
     test_generic(bs)
     assert len(bs.section_band_gap) == 1
     assert bs.section_band_gap[0].value == 0
 
     # Polarized, no gaps
-    bs = bands_polarized_no_gap.entry_archive.section_run[0].section_single_configuration_calculation[0].section_k_band[0]
+    bs = bands_polarized_no_gap.section_run[0].section_single_configuration_calculation[0].section_k_band[0]
     test_generic(bs)
     assert len(bs.section_band_gap) == 2
     assert bs.section_band_gap[0].value == 0
     assert bs.section_band_gap[1].value == 0