From 8677c7fbe4fa05930a7fb5508fdfd099b0965bf8 Mon Sep 17 00:00:00 2001
From: Markus Scheidgen <markus.scheidgen@gmail.com>
Date: Sun, 27 Jan 2019 15:11:40 +0100
Subject: [PATCH] Completed migration tests.
---
nomad/api/app.py | 2 +
nomad/api/upload.py | 15 +--
nomad/migration.py | 68 ++++++-----
nomad/processing/data.py | 2 +-
tests/bravado_flaks.py | 3 +-
.../migration/archive/upload/archive.tar.gz | Bin 0 -> 825 bytes
.../failed_calc/upload/1/template.json | 110 ++++++++++++++++++
.../failed_calc/upload/2/template.json | 4 +
.../failed_upload/upload/archive.tar.gz | 1 +
.../missing_calc/upload/1/template.json | 110 ++++++++++++++++++
.../missmatch/upload/1/template.json | 110 ++++++++++++++++++
.../missmatch/upload/2/template.json | 110 ++++++++++++++++++
.../migration/new_calc/upload/1/template.json | 110 ++++++++++++++++++
.../migration/new_calc/upload/2/template.json | 110 ++++++++++++++++++
.../migration/new_calc/upload/3/template.json | 110 ++++++++++++++++++
.../new_upload/new_upload/1/template.json | 110 ++++++++++++++++++
.../new_upload/new_upload/2/template.json | 110 ++++++++++++++++++
tests/test_migration.py | 48 ++++++--
18 files changed, 1073 insertions(+), 60 deletions(-)
create mode 100644 tests/data/migration/archive/upload/archive.tar.gz
create mode 100644 tests/data/migration/failed_calc/upload/1/template.json
create mode 100644 tests/data/migration/failed_calc/upload/2/template.json
create mode 100644 tests/data/migration/failed_upload/upload/archive.tar.gz
create mode 100644 tests/data/migration/missing_calc/upload/1/template.json
create mode 100644 tests/data/migration/missmatch/upload/1/template.json
create mode 100644 tests/data/migration/missmatch/upload/2/template.json
create mode 100644 tests/data/migration/new_calc/upload/1/template.json
create mode 100644 tests/data/migration/new_calc/upload/2/template.json
create mode 100644 tests/data/migration/new_calc/upload/3/template.json
create mode 100644 tests/data/migration/new_upload/new_upload/1/template.json
create mode 100644 tests/data/migration/new_upload/new_upload/2/template.json
diff --git a/nomad/api/app.py b/nomad/api/app.py
index fee8249d80..69bc14a224 100644
--- a/nomad/api/app.py
+++ b/nomad/api/app.py
@@ -66,6 +66,8 @@ api = Api(
@api.errorhandler
def handle(error: Exception):
status_code = getattr(error, 'code', 500)
+ if not isinstance(status_code, int):
+ status_code = 500
name = getattr(error, 'name', 'Internal Server Error')
description = getattr(error, 'description', 'No description available')
data = dict(
diff --git a/nomad/api/upload.py b/nomad/api/upload.py
index 9fbf452747..1beb01df1b 100644
--- a/nomad/api/upload.py
+++ b/nomad/api/upload.py
@@ -172,24 +172,13 @@ class UploadListResource(Resource):
if upload.name is None or upload.name is '':
upload.name = file.filename
- upload_files = ArchiveBasedStagingUploadFiles(
- upload.upload_id, create=True, local_path=local_path,
- file_name='.upload.%s' % os.path.splitext(file.filename)[1])
+ upload_files = ArchiveBasedStagingUploadFiles(upload.upload_id, create=True)
file.save(upload_files.upload_file_os_path)
else:
# simple streaming data in HTTP body, e.g. with curl "url" -T local_file
- file_name = '.upload'
- try:
- ext = os.path.splitext(upload.name)[1]
- if ext is not None:
- file_name += '.' + ext
- except Exception:
- pass
- upload_files = ArchiveBasedStagingUploadFiles(
- upload.upload_id, create=True, local_path=local_path,
- file_name='.upload')
+ upload_files = ArchiveBasedStagingUploadFiles(upload.upload_id, create=True)
try:
with open(upload_files.upload_file_os_path, 'wb') as f:
diff --git a/nomad/migration.py b/nomad/migration.py
index 3490893150..32e60ca481 100644
--- a/nomad/migration.py
+++ b/nomad/migration.py
@@ -91,7 +91,11 @@ class SourceCalc(Document):
total = source_query.count()
while True:
- calcs = source_query.filter(Calc.coe_calc_id > start_pid).order_by(Calc.coe_calc_id).limit(per_query)
+ calcs = source_query \
+ .filter(Calc.coe_calc_id > start_pid) \
+ .order_by(Calc.coe_calc_id) \
+ .limit(per_query)
+
source_calcs = []
for calc in calcs:
if calc.calc_metadata is None or calc.calc_metadata.filenames is None:
@@ -269,11 +273,12 @@ class NomadCOEMigration:
source_upload_id = os.path.split(os.path.split(upload_path)[0])[1]
upload_name = os.path.basename(upload_path)
else:
- potential_upload_archive = os.path.join(upload_path, NomadCOEMigration.archive_filename)
+ potential_upload_archive = os.path.join(
+ upload_path, NomadCOEMigration.archive_filename)
if os.path.isfile(potential_upload_archive):
upload_archive_f = open(potential_upload_archive, 'rb')
source_upload_id = os.path.split(os.path.split(potential_upload_archive)[0])[1]
- upload_name = '%s.tar.gz' % source_upload_id
+ upload_name = os.path.basename(potential_upload_archive)
else:
source_upload_id = os.path.split(upload_path)[1]
zip_file = zipstream.ZipFile()
@@ -289,22 +294,13 @@ class NomadCOEMigration:
upload_name = '%s.zip' % source_upload_id
# upload and process the upload file
-
- upload = self.client.uploads.upload(file=upload_archive_f, name=upload_name).response().result
+ upload = self.client.uploads.upload(
+ file=upload_archive_f, name=upload_name).response().result
upload_archive_f.close()
upload_logger = self.logger.bind(
source_upload_id=source_upload_id, upload_id=upload.upload_id)
- while upload.tasks_running:
- upload = self.client.uploads.get_upload(upload_id=upload.upload_id).response().result
- time.sleep(0.1)
-
- if upload.tasks_status == FAILURE:
- error = 'failed to process upload'
- upload_logger.logger.error(error, process_errors=upload.errors)
- return dict(error=error, status=FAILURE)
-
# grab source metadata
upload_metadata_calcs = list()
metadata_dict = dict()
@@ -313,20 +309,34 @@ class NomadCOEMigration:
source_metadata = CalcWithMetadata(upload_id=upload.upload_id, **source_calc.metadata)
source_metadata.mainfile = source_calc.mainfile
source_metadata.pid = source_calc.pid
- source_metadata.__migrated = None
+ source_metadata.__migrated = False
upload_metadata_calcs.append(source_metadata)
metadata_dict[source_calc.mainfile] = source_metadata
- # verify upload
report = utils.POPO()
- report.total_calcs = upload.calcs.pagination.total
report.total_source_calcs = len(metadata_dict)
- report.unprocessed_calcs = 0
+ report.failed_calcs = 0
report.migrated_calcs = 0
report.calcs_with_diffs = 0
report.new_calcs = 0
report.missing_calcs = 0
+ # wait for complete upload
+ while upload.tasks_running:
+ upload = self.client.uploads.get_upload(upload_id=upload.upload_id).response().result
+ time.sleep(0.1)
+
+ if upload.tasks_status == FAILURE:
+ error = 'failed to process upload'
+ report.missing_calcs = report.total_source_calcs
+ report.total_calcs = 0
+ upload_logger.error(error, process_errors=upload.errors)
+ yield report
+ continue
+ else:
+ report.total_calcs = upload.calcs.pagination.total
+
+ # verify upload
for page in range(1, math.ceil(report.total_calcs / 100) + 1):
upload = self.client.uploads.get_upload(
upload_id=upload.upload_id, per_page=100, page=page,
@@ -351,22 +361,17 @@ class NomadCOEMigration:
upload.upload_id, calc_proc.calc_id, source_calc, calc_logger):
report.calcs_with_diffs += 1
else:
- report.unprocessed_calcs += 1
- calc_logger.info(
- 'could not process a calc%s.' %
- ', that is in source' if source_calc is not None else '')
-
- if source_calc is not None:
- source_calc.__migrated = False
-
+ report.failed_calcs += 1
+ calc_logger.error(
+ 'could not process a calc', process_errors=calc_proc.errors)
continue
for source_calc in upload_metadata_calcs:
- if source_calc.__migrated is None:
+ if source_calc.__migrated is False:
report.missing_calcs += 1
- upload_logger.info('no match for source calc', mainfile=source_calc.mainfile)
- elif source_calc.__migrated is False:
- upload_logger.info('source calc not processed', mainfile=source_calc.mainfile)
+ upload_logger.info(
+ 'no match or processed calc for source calc',
+ mainfile=source_calc.mainfile)
# commit upload
admin_keys = ['upload_time, uploader, pid']
@@ -388,7 +393,7 @@ class NomadCOEMigration:
transform(calc) for calc in upload_metadata['calculations']
if calc.__migrated]
- if report.total_calcs > report.unprocessed_calcs:
+ if report.total_calcs > report.failed_calcs:
upload = self.client.uploads.exec_upload_command(
upload_id=upload.upload_id,
payload=dict(command='commit', metadata=upload_metadata)
@@ -405,7 +410,6 @@ class NomadCOEMigration:
# report
upload_logger.info('migrated upload', **report)
- report.update(status=SUCCESS)
yield report
def index(self, *args, **kwargs):
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index 7e0b47dfa4..9ddd11b05e 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -495,4 +495,4 @@ class Upload(Chord, datamodel.Upload):
@property
def calcs(self):
- return Calc.objects(upload_id=self.upload_id)
+ return Calc.objects(upload_id=self.upload_id, tasks_status=SUCCESS)
diff --git a/tests/bravado_flaks.py b/tests/bravado_flaks.py
index 656f56ede4..e5d664e4ad 100644
--- a/tests/bravado_flaks.py
+++ b/tests/bravado_flaks.py
@@ -92,9 +92,8 @@ class FlaskTestFutureAdapter:
if len(files) > 1:
raise NotImplementedError
if len(files) == 1:
- data = dict() if data is None else data
_, (_, f) = files[0]
- data.update(file=(f, 'file'))
+ data = f
return function(
url, headers=self._request_params.get('headers'), data=data)
diff --git a/tests/data/migration/archive/upload/archive.tar.gz b/tests/data/migration/archive/upload/archive.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..a665e418544655927532e20d5c5b3d3c2250e573
GIT binary patch
literal 825
zcmb2|=3tOo;v2)j{5Cw=|F(h1JLCO_P8qk{@SKy9e}K*DHuI$mEhY<3EXkX$+i7f_
zSkm#|ZaHu4qh01Mk{SQv?%eoy?&HoMSJ>tF%J_~KGTmptEO_3Hy>jzL)`oTMG2#aq
zb{}XIc$~oS=dOIN!3tN`9X4KmKl-C|?yGO_b<=*brR>acbEb-ji+j#^xW0YgBx-$8
z@o%GTe&(Or^Ixa_sGj$jw_)?-%Njq<Kd#@;uA<{OSu54`%<Dr{?EC!G8F%En?^M+D
z@Ov7R|GX}}wkD3}<=i_DOVYg09h>u{_|x>o=f$U`{hd~%*lHGSBYoRQzM(p#NS%dq
z@=ee83>uv3k#5ExG<9F?{(M<@?i^{QAVKb^qgR*A$(ph<HsC||ouG@VHc=BVzmk?N
z^LaOWt@&Qnx7xRN$-U6O=woE5eMv=3bp7O2&%UhBdj0q1Qmr?ESFd{7tm=N5KZ*0{
zobL3i^TP!<tzY<Jk$L>vkIyfiKiBtY^^ZqCnwJ{LRo%Z{y=!W=ug(#d6H!}Ib)qh<
ziue-dGVA)LTc%f9(q<l8{K}kRpYH0Q1_P%r3|TTyZv660eHV3i%L-5JOy;lWmv~yg
zym@xNdEX&}S&A8b3$xxFk14D=)D>v8?&q2Js!uhgPwh7FDULGv9yk5Qr+EQi4NgA`
z(^3~b{diL6mZ(LU7cEclIZGsoAJS#&%Q?>5bfMd3ed&~Q{{B5@b_u@?`T6<j_bTIS
zR&}e7%sytPY#tJAy*7DY0MFX~?%?^sZ|?-3j*03qmiqqQ#c9)$wcfYI4wOFHrqR1v
z^{8m}>D)6fGlCf>U72e5`+BregV>aQ^^YBwj~!vo_&u?kS>^yom7vp!>5+o3EJas)
zn5myEY7J~D6#3b>yG<nCiue3hK^KYkN!-fr*DPAh_%$c?i)8+}JWp7w`g4Qsimk_{
zoaDFbjEr1b`Kx*wleDzaq<#&p`~LRXQ7R?g)*_#drG2TqtW#~`GhdBMG1=%~+AY`9
z0r|T&pNzfx`kYBT|Nmcw|Ck)sZ~4zJbNT;%mJ8W$|JQ;TKL7K7)nEQ!p7`s(9h5!$
d-~BKDPdjIU#HA%5U=Qyf#`5B&9Sj-_3;?9qoooOA
literal 0
HcmV?d00001
diff --git a/tests/data/migration/failed_calc/upload/1/template.json b/tests/data/migration/failed_calc/upload/1/template.json
new file mode 100644
index 0000000000..b85b5bd174
--- /dev/null
+++ b/tests/data/migration/failed_calc/upload/1/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/failed_calc/upload/2/template.json b/tests/data/migration/failed_calc/upload/2/template.json
new file mode 100644
index 0000000000..108dd63a26
--- /dev/null
+++ b/tests/data/migration/failed_calc/upload/2/template.json
@@ -0,0 +1,4 @@
+{
+ "section_run": [
+ {
+ not parsable
\ No newline at end of file
diff --git a/tests/data/migration/failed_upload/upload/archive.tar.gz b/tests/data/migration/failed_upload/upload/archive.tar.gz
new file mode 100644
index 0000000000..9f53f813bb
--- /dev/null
+++ b/tests/data/migration/failed_upload/upload/archive.tar.gz
@@ -0,0 +1 @@
+not a tar.gz
diff --git a/tests/data/migration/missing_calc/upload/1/template.json b/tests/data/migration/missing_calc/upload/1/template.json
new file mode 100644
index 0000000000..b85b5bd174
--- /dev/null
+++ b/tests/data/migration/missing_calc/upload/1/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/missmatch/upload/1/template.json b/tests/data/migration/missmatch/upload/1/template.json
new file mode 100644
index 0000000000..b85b5bd174
--- /dev/null
+++ b/tests/data/migration/missmatch/upload/1/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/missmatch/upload/2/template.json b/tests/data/migration/missmatch/upload/2/template.json
new file mode 100644
index 0000000000..4a2b1b571e
--- /dev/null
+++ b/tests/data/migration/missmatch/upload/2/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "missmatch"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/new_calc/upload/1/template.json b/tests/data/migration/new_calc/upload/1/template.json
new file mode 100644
index 0000000000..b85b5bd174
--- /dev/null
+++ b/tests/data/migration/new_calc/upload/1/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/new_calc/upload/2/template.json b/tests/data/migration/new_calc/upload/2/template.json
new file mode 100644
index 0000000000..b85b5bd174
--- /dev/null
+++ b/tests/data/migration/new_calc/upload/2/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/new_calc/upload/3/template.json b/tests/data/migration/new_calc/upload/3/template.json
new file mode 100644
index 0000000000..b85b5bd174
--- /dev/null
+++ b/tests/data/migration/new_calc/upload/3/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/new_upload/new_upload/1/template.json b/tests/data/migration/new_upload/new_upload/1/template.json
new file mode 100644
index 0000000000..b85b5bd174
--- /dev/null
+++ b/tests/data/migration/new_upload/new_upload/1/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/new_upload/new_upload/2/template.json b/tests/data/migration/new_upload/new_upload/2/template.json
new file mode 100644
index 0000000000..b85b5bd174
--- /dev/null
+++ b/tests/data/migration/new_upload/new_upload/2/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/test_migration.py b/tests/test_migration.py
index 6dc1de5369..e3360e3b07 100644
--- a/tests/test_migration.py
+++ b/tests/test_migration.py
@@ -14,13 +14,14 @@
import pytest
import os
+import os.path
from bravado.client import SwaggerClient
+import json
from nomad import infrastructure, coe_repo
from nomad.migration import NomadCOEMigration, SourceCalc
from nomad.infrastructure import repository_db_connection
-from nomad.processing import SUCCESS
from .bravado_flaks import FlaskTestHttpClient
from tests.conftest import create_repository_db
@@ -153,17 +154,40 @@ def migrate_infra(migration, target_repo, flask_client, worker, monkeysession):
monkeysession.setattr('nomad.infrastructure.repository_db', old_repo)
-@pytest.mark.parametrize('upload, assertions', [('baseline', dict(migrated=2))])
+mirgation_test_specs = [
+ ('baseline', dict(migrated=2, source=2)),
+ ('archive', dict(migrated=2, source=2)),
+ ('new_upload', dict(new=2)),
+ ('new_calc', dict(migrated=2, source=2, new=1)),
+ ('missing_calc', dict(migrated=1, source=2, missing=1)),
+ ('missmatch', dict(migrated=2, source=2, diffs=1)),
+ ('failed_calc', dict(migrated=1, source=2, diffs=0, missing=1, failed=1, errors=1)),
+ ('failed_upload', dict(migrated=0, source=2, missing=2, errors=1))
+]
+
+
+@pytest.mark.parametrize('test, assertions', mirgation_test_specs)
@pytest.mark.timeout(10)
-def test_migrate(migrate_infra, upload, assertions):
- upload_path = os.path.join('tests', 'data', 'migration', upload, 'upload')
- reports = list(migrate_infra.migrate(upload_path))
+def test_migrate(migrate_infra, test, assertions, caplog):
+ uploads_path = os.path.join('tests', 'data', 'migration', test)
+ reports = list(migrate_infra.migrate(
+ *[os.path.join(uploads_path, dir) for dir in os.listdir(uploads_path)]))
+
assert len(reports) == 1
report = reports[0]
- assert report['status'] == SUCCESS
- assert report['total_calcs'] == 2
- assert report['total_source_calcs'] == 2
- assert report['migrated_calcs'] == 2
- assert report['calcs_with_diffs'] == 0
- assert report['new_calcs'] == 0
- assert report['missing_calcs'] == 0
+ assert report['total_calcs'] == assertions.get('migrated', 0) + assertions.get('new', 0) + assertions.get('failed', 0)
+
+ assert report['total_source_calcs'] == assertions.get('source', 0)
+ assert report['migrated_calcs'] == assertions.get('migrated', 0)
+ assert report['calcs_with_diffs'] == assertions.get('diffs', 0)
+ assert report['new_calcs'] == assertions.get('new', 0)
+ assert report['missing_calcs'] == assertions.get('missing', 0)
+
+ errors = 0
+ for record in caplog.get_records(when='call'):
+ if record.levelname in ['ERROR', 'CRITICAL']:
+ record_data = json.loads(record.getMessage())
+ if 'source_upload_id' in record_data:
+ errors += 1
+
+ assert errors == assertions.get('errors', 0)
--
GitLab