Commit 80fe4236 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added artifical template parser.

parent 1c85fb8c
......@@ -44,7 +44,7 @@
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_parsing.py"
"-sv", "tests/test_parsing.py::test_parser[parsers/template-tests/data/template.json]"
]
},
{
......
......@@ -227,7 +227,9 @@ class Upload():
try:
zipFile = ZipFile(self.upload_file)
zipFile.extractall(self.upload_extract_dir)
self.filelist = [zipInfo.filename for zipInfo in zipFile.filelist]
self.filelist = [
zipInfo.filename for zipInfo in zipFile.filelist
if not zipInfo.filename.endswith('/')]
except BadZipFile as e:
raise UploadError('Upload is not a zip file', e)
finally:
......
......@@ -13,7 +13,7 @@
# limitations under the License.
"""
The *parsing* modules is currenlty an abstraction for the existin NOMAD-coe parsers.
The *parsing* module is an interface for the existing NOMAD-coe parsers.
The parser code is used via :mod:`nomad.dependencies`. This module redefines
some of the old NOMAD-coe python-common functionality to create a more coherent
interface to the parsers.
......@@ -47,9 +47,11 @@ Parsers in NOMAD-coe use a *backend* to create output.
from nomad.parsing.backend import AbstractParserBackend, LocalBackend, LegacyLocalBackend, JSONStreamWriter, BadContextURI, WrongContextState
from nomad.parsing.parser import Parser, LegacyParser
from nomad.parsing.artificial import TemplateParser
from nomad.dependencies import dependencies_dict as dependencies
parsers = [
TemplateParser(),
LegacyParser(
python_git=dependencies['parsers/vasp'],
parser_class_name='vaspparser.VASPParser',
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Parser for creating artificial test, brenchmark, and demonstration data.
"""
from typing import Callable, IO, Any
import json
import os.path
import numpy as np
from nomadcore.local_meta_info import loadJsonFile, InfoKindEl
from nomad.parsing.backend import LocalBackend
from nomad.parsing.parser import Parser
class TemplateParser(Parser):
"""
A parser that generates data based on a template given via the
mainfile. The template is basically some archive json. Only
"""
def __init__(self):
super().__init__()
# use vasp metainfo, not to really use it, but because it works
file_dir = os.path.dirname(os.path.abspath(__file__))
relative_metainfo_path = "../../.dependencies/nomad-meta-info/meta_info/nomad_meta_info/vasp.nomadmetainfo.json"
meta_info_path = os.path.normpath(os.path.join(file_dir, relative_metainfo_path))
self.meta_info_env, _ = loadJsonFile(filePath=meta_info_path, dependencyLoader=None, extraArgsHandling=InfoKindEl.ADD_EXTRA_ARGS, uri=None)
self.name = 'parsers/template'
self.backend = None
def is_mainfile(self, filename: str, open: Callable[[str], IO[Any]]) -> bool:
return filename.endswith('template.json')
def add_section(self, section):
if not isinstance(section, dict):
print(section)
name = section['_name']
index = self.backend.openSection(name)
for key, value in section.items():
if key.startswith('x_') or key.startswith('_'):
continue
if key.startswith('section_'):
values = value if isinstance(value, list) else [value]
for value in values:
self.add_section(value)
else:
if isinstance(value, list):
shape = self.meta_info_env[key].get('shape')
if shape is None or len(shape) == 0:
for single_value in value:
self.backend.addValue(key, single_value, index)
else:
self.backend.addArrayValues(key, np.asarray(value), index)
else:
self.backend.addValue(key, value, index)
self.backend.closeSection(name, index)
def run(self, mainfile: str) -> LocalBackend:
self.backend = LocalBackend(metaInfoEnv=self.meta_info_env, debug=False)
template_json = json.load(open(mainfile, 'r'))
section = template_json['section_run'][0]
self.add_section(section)
self.backend.finishedParsingSession('ParseSuccess', [])
return self.backend
import pytest
from mongoengine import connect
from mongoengine.connection import disconnect
import time
from nomad import config
......@@ -17,6 +18,18 @@ def celery_config():
}
@pytest.fixture(scope='function')
def worker(celery_session_worker):
"""
Extension of the buildin celery_session_worker fixture that adds sleep to consume
bleeding tasks.
Processes might be completed (and therefore the test it self) before child
processes are finished. Therefore open task request might bleed into the next test.
"""
yield
time.sleep(0.2)
@pytest.fixture(scope='function', autouse=True)
def mongomock(monkeypatch):
def mock_connect(**kwargs):
......
{
"section_run": [
{
"_name": "section_run",
"_gIndex": 0,
"program_name": "VASP",
"program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
"program_basis_set_type": "plane waves",
"program_compilation_datetime": 63536.0,
"section_method": [
{
"_name": "section_method",
"_gIndex": 0,
"electronic_structure_method": "DFT",
"section_XC_functionals": [
{
"_name": "section_XC_functionals",
"_gIndex": 0,
"XC_functional_name": "GGA_X_PBE"
},
{
"_name": "section_XC_functionals",
"_gIndex": 1,
"XC_functional_name": "GGA_C_PBE"
}
],
"section_method_basis_set": [
{
"_name": "section_method_basis_set",
"_gIndex": 0,
"mapping_section_method_basis_set_cell_associated": 0
}
],
"section_method_atom_kind": [
{
"_name": "section_method_atom_kind",
"_gIndex": 0,
"method_atom_kind_atom_number": 35,
"method_atom_kind_label": "Br",
"method_atom_kind_mass": 79.904,
"method_atom_kind_explicit_electrons": 7.0,
"method_atom_kind_pseudopotential_name": " PAW_PBE Br 06Sep2000 "
},
{
"_name": "section_method_atom_kind",
"_gIndex": 1,
"method_atom_kind_atom_number": 19,
"method_atom_kind_label": "K",
"method_atom_kind_mass": 39.098,
"method_atom_kind_explicit_electrons": 9.0,
"method_atom_kind_pseudopotential_name": " PAW_PBE K_sv 06Sep2000 "
},
{
"_name": "section_method_atom_kind",
"_gIndex": 2,
"method_atom_kind_atom_number": 14,
"method_atom_kind_label": "Si",
"method_atom_kind_mass": 28.085,
"method_atom_kind_explicit_electrons": 4.0,
"method_atom_kind_pseudopotential_name": " PAW_PBE Si 05Jan2001 "
}
]
}
],
"section_basis_set_cell_dependent": [
{
"_name": "section_basis_set_cell_dependent",
"_gIndex": 0,
"basis_set_planewave_cutoff": 5.81433707960672e-17
}
],
"section_system": [
{
"_name": "section_system",
"_gIndex": 1,
"configuration_raw_gid": "sX4Re16Q_-KSbCvIYPvikfbaxKxMe",
"atom_species": [
35,
19,
14,
14
],
"chemical_composition": "BrKSiSi",
"chemical_composition_reduced": "BrKSi2",
"chemical_composition_bulk_reduced": "BrKSi2",
"system_type": "Bulk",
"simulation_cell": [
[
5.76372622e-10,
0.0,
0.0
],
[
0.0,
5.76372622e-10,
0.0
],
[
0.0,
0.0,
4.0755698899999997e-10
]
],
"configuration_periodic_dimensions": [
true,
true,
true
],
"atom_positions": [
[
2.88186311e-10,
0.0,
2.0377849449999999e-10
],
[
0.0,
2.88186311e-10,
2.0377849449999999e-10
],
[
0.0,
0.0,
0.0
],
[
2.88186311e-10,
2.88186311e-10,
0.0
]
],
"atom_labels": [
"Br",
"K",
"Si",
"Si"
]
}
],
"section_single_configuration_calculation": [
{
"_name": "section_single_configuration_calculation",
"_gIndex": 0,
"single_configuration_calculation_to_system_ref": 0,
"single_configuration_to_calculation_method_ref": 0,
"energy_free": -1.5936767191492225e-18,
"energy_total": -1.5935696296699573e-18,
"energy_total_T0": -3.2126683561907e-22
}
],
"section_sampling_method": [
{
"_name": "section_sampling_method",
"_gIndex": 0,
"sampling_method": "geometry_optimization"
}
],
"section_frame_sequence": [
{
"_name": "section_frame_sequence",
"_gIndex": 0,
"frame_sequence_to_sampling_ref": 0,
"frame_sequence_local_frames_ref": [
0
]
}
]
}
]
}
\ No newline at end of file
......@@ -122,12 +122,9 @@ class ChildProc(Proc):
self.parent.on_child_complete()
def test_counter(celery_session_worker):
def test_counter(worker):
p = ParentProc.create()
p.spawn_children()
p.block_until_complete()
assert_proc(p, 'after_children')
# wait for session worker to complete all open tasks
# otherwise uncompleted task request will bleed into the next tests
time.sleep(1)
......@@ -79,13 +79,13 @@ def assert_processing(upload: Upload):
@pytest.mark.timeout(30)
def test_processing(uploaded_id, celery_session_worker):
def test_processing(uploaded_id, worker):
upload = run_processing(uploaded_id)
assert_processing(upload)
@pytest.mark.parametrize('uploaded_id', [example_files[1]], indirect=True)
def test_processing_doublets(uploaded_id, celery_session_worker, caplog):
def test_processing_doublets(uploaded_id, worker, caplog):
caplog.set_level(logging.CRITICAL)
upload = run_processing(uploaded_id)
......@@ -99,7 +99,7 @@ def test_processing_doublets(uploaded_id, celery_session_worker, caplog):
@pytest.mark.timeout(30)
def test_process_non_existing(celery_session_worker, caplog):
def test_process_non_existing(worker, caplog):
caplog.set_level(logging.CRITICAL)
upload = run_processing('__does_not_exist')
......@@ -111,7 +111,7 @@ def test_process_non_existing(celery_session_worker, caplog):
@pytest.mark.parametrize('task', ['extracting', 'parse_all', 'cleanup', 'parsing'])
@pytest.mark.timeout(30)
def test_task_failure(monkeypatch, uploaded_id, celery_session_worker, task, caplog):
def test_task_failure(monkeypatch, uploaded_id, worker, task, caplog):
caplog.set_level(logging.CRITICAL)
# mock the task method to through exceptions
......
......@@ -159,7 +159,7 @@ def test_upload_to_upload(client, file):
@pytest.mark.parametrize("file", example_files)
@pytest.mark.timeout(10)
def test_processing(client, file, celery_session_worker, mocksearch):
def test_processing(client, file, worker, mocksearch):
handler = handle_uploads_thread(quit=True)
rv = client.post('/uploads')
......
......@@ -27,7 +27,9 @@ from minio.error import NoSuchKey, NoSuchBucket
import nomad.files as files
import nomad.config as config
example_file = 'tests/data/proc/examples_vasp.zip'
# example_file uses an artificial parser for faster test execution, can also be
# changed to examples_vasp.zip for using vasp parser
example_file = 'tests/data/proc/examples_template.zip'
empty_file = 'tests/data/proc/empty.zip'
......
......@@ -23,6 +23,7 @@ from nomad.parsing import JSONStreamWriter, parser_dict
from nomad.parsing import LocalBackend, BadContextURI
parser_examples = [
('parsers/template', 'tests/data/parsers/template.json'),
('parsers/exciting', '.dependencies/parsers/exciting/test/examples/Ag/INFO.OUT'),
('parsers/exciting', '.dependencies/parsers/exciting/test/examples/GW/INFO.OUT'),
('parsers/vasp', '.dependencies/parsers/vasp/test/examples/xml/perovskite.xml'),
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment