diff --git a/nomad/api/app.py b/nomad/api/app.py
index fee8249d809fae4653c6b25078821a97ef4b183d..69bc14a22487ab3d262b5b7d818e49a09afb046c 100644
--- a/nomad/api/app.py
+++ b/nomad/api/app.py
@@ -66,6 +66,8 @@ api = Api(
@api.errorhandler
def handle(error: Exception):
status_code = getattr(error, 'code', 500)
+ if not isinstance(status_code, int):
+ status_code = 500
name = getattr(error, 'name', 'Internal Server Error')
description = getattr(error, 'description', 'No description available')
data = dict(
diff --git a/nomad/api/upload.py b/nomad/api/upload.py
index 9fbf4527478e6fa719e19dfcaf74dd0c9e9d544b..1beb01df1ba817126799c4cb46861a296ee83a46 100644
--- a/nomad/api/upload.py
+++ b/nomad/api/upload.py
@@ -172,24 +172,13 @@ class UploadListResource(Resource):
if upload.name is None or upload.name is '':
upload.name = file.filename
- upload_files = ArchiveBasedStagingUploadFiles(
- upload.upload_id, create=True, local_path=local_path,
- file_name='.upload.%s' % os.path.splitext(file.filename)[1])
+ upload_files = ArchiveBasedStagingUploadFiles(upload.upload_id, create=True)
file.save(upload_files.upload_file_os_path)
else:
# simple streaming data in HTTP body, e.g. with curl "url" -T local_file
- file_name = '.upload'
- try:
- ext = os.path.splitext(upload.name)[1]
- if ext is not None:
- file_name += '.' + ext
- except Exception:
- pass
- upload_files = ArchiveBasedStagingUploadFiles(
- upload.upload_id, create=True, local_path=local_path,
- file_name='.upload')
+ upload_files = ArchiveBasedStagingUploadFiles(upload.upload_id, create=True)
try:
with open(upload_files.upload_file_os_path, 'wb') as f:
diff --git a/nomad/migration.py b/nomad/migration.py
index 349089315026ac4dc0ed82cff10d647db8e6fcd2..32e60ca4818f2e2455d7ef2e8b0332f702953b07 100644
--- a/nomad/migration.py
+++ b/nomad/migration.py
@@ -91,7 +91,11 @@ class SourceCalc(Document):
total = source_query.count()
while True:
- calcs = source_query.filter(Calc.coe_calc_id > start_pid).order_by(Calc.coe_calc_id).limit(per_query)
+ calcs = source_query \
+ .filter(Calc.coe_calc_id > start_pid) \
+ .order_by(Calc.coe_calc_id) \
+ .limit(per_query)
+
source_calcs = []
for calc in calcs:
if calc.calc_metadata is None or calc.calc_metadata.filenames is None:
@@ -269,11 +273,12 @@ class NomadCOEMigration:
source_upload_id = os.path.split(os.path.split(upload_path)[0])[1]
upload_name = os.path.basename(upload_path)
else:
- potential_upload_archive = os.path.join(upload_path, NomadCOEMigration.archive_filename)
+ potential_upload_archive = os.path.join(
+ upload_path, NomadCOEMigration.archive_filename)
if os.path.isfile(potential_upload_archive):
upload_archive_f = open(potential_upload_archive, 'rb')
source_upload_id = os.path.split(os.path.split(potential_upload_archive)[0])[1]
- upload_name = '%s.tar.gz' % source_upload_id
+ upload_name = os.path.basename(potential_upload_archive)
else:
source_upload_id = os.path.split(upload_path)[1]
zip_file = zipstream.ZipFile()
@@ -289,22 +294,13 @@ class NomadCOEMigration:
upload_name = '%s.zip' % source_upload_id
# upload and process the upload file
-
- upload = self.client.uploads.upload(file=upload_archive_f, name=upload_name).response().result
+ upload = self.client.uploads.upload(
+ file=upload_archive_f, name=upload_name).response().result
upload_archive_f.close()
upload_logger = self.logger.bind(
source_upload_id=source_upload_id, upload_id=upload.upload_id)
- while upload.tasks_running:
- upload = self.client.uploads.get_upload(upload_id=upload.upload_id).response().result
- time.sleep(0.1)
-
- if upload.tasks_status == FAILURE:
- error = 'failed to process upload'
- upload_logger.logger.error(error, process_errors=upload.errors)
- return dict(error=error, status=FAILURE)
-
# grab source metadata
upload_metadata_calcs = list()
metadata_dict = dict()
@@ -313,20 +309,34 @@ class NomadCOEMigration:
source_metadata = CalcWithMetadata(upload_id=upload.upload_id, **source_calc.metadata)
source_metadata.mainfile = source_calc.mainfile
source_metadata.pid = source_calc.pid
- source_metadata.__migrated = None
+ source_metadata.__migrated = False
upload_metadata_calcs.append(source_metadata)
metadata_dict[source_calc.mainfile] = source_metadata
- # verify upload
report = utils.POPO()
- report.total_calcs = upload.calcs.pagination.total
report.total_source_calcs = len(metadata_dict)
- report.unprocessed_calcs = 0
+ report.failed_calcs = 0
report.migrated_calcs = 0
report.calcs_with_diffs = 0
report.new_calcs = 0
report.missing_calcs = 0
+ # wait for complete upload
+ while upload.tasks_running:
+ upload = self.client.uploads.get_upload(upload_id=upload.upload_id).response().result
+ time.sleep(0.1)
+
+ if upload.tasks_status == FAILURE:
+ error = 'failed to process upload'
+ report.missing_calcs = report.total_source_calcs
+ report.total_calcs = 0
+ upload_logger.error(error, process_errors=upload.errors)
+ yield report
+ continue
+ else:
+ report.total_calcs = upload.calcs.pagination.total
+
+ # verify upload
for page in range(1, math.ceil(report.total_calcs / 100) + 1):
upload = self.client.uploads.get_upload(
upload_id=upload.upload_id, per_page=100, page=page,
@@ -351,22 +361,17 @@ class NomadCOEMigration:
upload.upload_id, calc_proc.calc_id, source_calc, calc_logger):
report.calcs_with_diffs += 1
else:
- report.unprocessed_calcs += 1
- calc_logger.info(
- 'could not process a calc%s.' %
- ', that is in source' if source_calc is not None else '')
-
- if source_calc is not None:
- source_calc.__migrated = False
-
+ report.failed_calcs += 1
+ calc_logger.error(
+ 'could not process a calc', process_errors=calc_proc.errors)
continue
for source_calc in upload_metadata_calcs:
- if source_calc.__migrated is None:
+ if source_calc.__migrated is False:
report.missing_calcs += 1
- upload_logger.info('no match for source calc', mainfile=source_calc.mainfile)
- elif source_calc.__migrated is False:
- upload_logger.info('source calc not processed', mainfile=source_calc.mainfile)
+ upload_logger.info(
+ 'no match or processed calc for source calc',
+ mainfile=source_calc.mainfile)
# commit upload
admin_keys = ['upload_time, uploader, pid']
@@ -388,7 +393,7 @@ class NomadCOEMigration:
transform(calc) for calc in upload_metadata['calculations']
if calc.__migrated]
- if report.total_calcs > report.unprocessed_calcs:
+ if report.total_calcs > report.failed_calcs:
upload = self.client.uploads.exec_upload_command(
upload_id=upload.upload_id,
payload=dict(command='commit', metadata=upload_metadata)
@@ -405,7 +410,6 @@ class NomadCOEMigration:
# report
upload_logger.info('migrated upload', **report)
- report.update(status=SUCCESS)
yield report
def index(self, *args, **kwargs):
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index 7e0b47dfa42f46cb1bc66e9df1dead56f5ab14bb..9ddd11b05e431c0e3de9f285fb6cf947b366ddd8 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -495,4 +495,4 @@ class Upload(Chord, datamodel.Upload):
@property
def calcs(self):
- return Calc.objects(upload_id=self.upload_id)
+ return Calc.objects(upload_id=self.upload_id, tasks_status=SUCCESS)
diff --git a/tests/bravado_flaks.py b/tests/bravado_flaks.py
index 656f56ede45fe47163387223c375e24a7f17d232..e5d664e4ad280916c70ac93c6ea0198d8502aeb8 100644
--- a/tests/bravado_flaks.py
+++ b/tests/bravado_flaks.py
@@ -92,9 +92,8 @@ class FlaskTestFutureAdapter:
if len(files) > 1:
raise NotImplementedError
if len(files) == 1:
- data = dict() if data is None else data
_, (_, f) = files[0]
- data.update(file=(f, 'file'))
+ data = f
return function(
url, headers=self._request_params.get('headers'), data=data)
diff --git a/tests/data/migration/archive/upload/archive.tar.gz b/tests/data/migration/archive/upload/archive.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..a665e418544655927532e20d5c5b3d3c2250e573
Binary files /dev/null and b/tests/data/migration/archive/upload/archive.tar.gz differ
diff --git a/tests/data/migration/failed_calc/upload/1/template.json b/tests/data/migration/failed_calc/upload/1/template.json
new file mode 100644
index 0000000000000000000000000000000000000000..b85b5bd1742a1b1591c39adbd9086565e9ec0cc3
--- /dev/null
+++ b/tests/data/migration/failed_calc/upload/1/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/failed_calc/upload/2/template.json b/tests/data/migration/failed_calc/upload/2/template.json
new file mode 100644
index 0000000000000000000000000000000000000000..108dd63a261cc4717db8568bd04d62e373aa4312
--- /dev/null
+++ b/tests/data/migration/failed_calc/upload/2/template.json
@@ -0,0 +1,4 @@
+{
+ "section_run": [
+ {
+ not parsable
\ No newline at end of file
diff --git a/tests/data/migration/failed_upload/upload/archive.tar.gz b/tests/data/migration/failed_upload/upload/archive.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..9f53f813bbf0d336c992e86d771b2b44b1cc39f3
--- /dev/null
+++ b/tests/data/migration/failed_upload/upload/archive.tar.gz
@@ -0,0 +1 @@
+not a tar.gz
diff --git a/tests/data/migration/missing_calc/upload/1/template.json b/tests/data/migration/missing_calc/upload/1/template.json
new file mode 100644
index 0000000000000000000000000000000000000000..b85b5bd1742a1b1591c39adbd9086565e9ec0cc3
--- /dev/null
+++ b/tests/data/migration/missing_calc/upload/1/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/missmatch/upload/1/template.json b/tests/data/migration/missmatch/upload/1/template.json
new file mode 100644
index 0000000000000000000000000000000000000000..b85b5bd1742a1b1591c39adbd9086565e9ec0cc3
--- /dev/null
+++ b/tests/data/migration/missmatch/upload/1/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/missmatch/upload/2/template.json b/tests/data/migration/missmatch/upload/2/template.json
new file mode 100644
index 0000000000000000000000000000000000000000..4a2b1b571e9214d9bd033b1baa97d9adb4e8c317
--- /dev/null
+++ b/tests/data/migration/missmatch/upload/2/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "missmatch"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/new_calc/upload/1/template.json b/tests/data/migration/new_calc/upload/1/template.json
new file mode 100644
index 0000000000000000000000000000000000000000..b85b5bd1742a1b1591c39adbd9086565e9ec0cc3
--- /dev/null
+++ b/tests/data/migration/new_calc/upload/1/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/new_calc/upload/2/template.json b/tests/data/migration/new_calc/upload/2/template.json
new file mode 100644
index 0000000000000000000000000000000000000000..b85b5bd1742a1b1591c39adbd9086565e9ec0cc3
--- /dev/null
+++ b/tests/data/migration/new_calc/upload/2/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/new_calc/upload/3/template.json b/tests/data/migration/new_calc/upload/3/template.json
new file mode 100644
index 0000000000000000000000000000000000000000..b85b5bd1742a1b1591c39adbd9086565e9ec0cc3
--- /dev/null
+++ b/tests/data/migration/new_calc/upload/3/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/new_upload/new_upload/1/template.json b/tests/data/migration/new_upload/new_upload/1/template.json
new file mode 100644
index 0000000000000000000000000000000000000000..b85b5bd1742a1b1591c39adbd9086565e9ec0cc3
--- /dev/null
+++ b/tests/data/migration/new_upload/new_upload/1/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/data/migration/new_upload/new_upload/2/template.json b/tests/data/migration/new_upload/new_upload/2/template.json
new file mode 100644
index 0000000000000000000000000000000000000000..b85b5bd1742a1b1591c39adbd9086565e9ec0cc3
--- /dev/null
+++ b/tests/data/migration/new_upload/new_upload/2/template.json
@@ -0,0 +1,110 @@
+{
+ "section_run": [
+ {
+ "_name": "section_run",
+ "_gIndex": 0,
+ "program_name": "VASP",
+ "program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
+ "program_basis_set_type": "plane waves",
+ "section_method": [
+ {
+ "_name": "section_method",
+ "_gIndex": 0,
+
+ "electronic_structure_method": "DFT",
+ "section_XC_functionals": [
+ {
+ "_name": "section_XC_functionals",
+ "_gIndex": 0,
+ "XC_functional_name": "GGA_X_PBE"
+ }
+ ]
+ }
+ ],
+ "section_system": [
+ {
+ "_name": "section_system",
+ "_gIndex": 0,
+ "simulation_cell": [
+ [
+ 5.76372622e-10,
+ 0.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 5.76372622e-10,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0,
+ 4.0755698899999997e-10
+ ]
+ ],
+ "configuration_periodic_dimensions": [
+ true,
+ true,
+ true
+ ],
+ "atom_positions": [
+ [
+ 2.88186311e-10,
+ 0.0,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 2.88186311e-10,
+ 2.0377849449999999e-10
+ ],
+ [
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ [
+ 2.88186311e-10,
+ 2.88186311e-10,
+ 0.0
+ ]
+ ],
+ "atom_labels": [
+ "Br",
+ "K",
+ "Si",
+ "Si"
+ ]
+ }
+ ],
+ "section_single_configuration_calculation": [
+ {
+ "_name": "section_single_configuration_calculation",
+ "_gIndex": 0,
+ "single_configuration_calculation_to_system_ref": 0,
+ "single_configuration_to_calculation_method_ref": 0,
+ "energy_free": -1.5936767191492225e-18,
+ "energy_total": -1.5935696296699573e-18,
+ "energy_total_T0": -3.2126683561907e-22
+ }
+ ],
+ "section_sampling_method": [
+ {
+ "_name": "section_sampling_method",
+ "_gIndex": 0,
+ "sampling_method": "geometry_optimization"
+ }
+ ],
+ "section_frame_sequence": [
+ {
+ "_name": "section_frame_sequence",
+ "_gIndex": 0,
+ "frame_sequence_to_sampling_ref": 0,
+ "frame_sequence_local_frames_ref": [
+ 0
+ ]
+ }
+ ]
+ }
+ ]
+ }
\ No newline at end of file
diff --git a/tests/test_migration.py b/tests/test_migration.py
index 6dc1de5369755b2878378b576618443a9479435a..e3360e3b078de5b0ab45a4d026622630ecf2799a 100644
--- a/tests/test_migration.py
+++ b/tests/test_migration.py
@@ -14,13 +14,14 @@
import pytest
import os
+import os.path
from bravado.client import SwaggerClient
+import json
from nomad import infrastructure, coe_repo
from nomad.migration import NomadCOEMigration, SourceCalc
from nomad.infrastructure import repository_db_connection
-from nomad.processing import SUCCESS
from .bravado_flaks import FlaskTestHttpClient
from tests.conftest import create_repository_db
@@ -153,17 +154,40 @@ def migrate_infra(migration, target_repo, flask_client, worker, monkeysession):
monkeysession.setattr('nomad.infrastructure.repository_db', old_repo)
-@pytest.mark.parametrize('upload, assertions', [('baseline', dict(migrated=2))])
+mirgation_test_specs = [
+ ('baseline', dict(migrated=2, source=2)),
+ ('archive', dict(migrated=2, source=2)),
+ ('new_upload', dict(new=2)),
+ ('new_calc', dict(migrated=2, source=2, new=1)),
+ ('missing_calc', dict(migrated=1, source=2, missing=1)),
+ ('missmatch', dict(migrated=2, source=2, diffs=1)),
+ ('failed_calc', dict(migrated=1, source=2, diffs=0, missing=1, failed=1, errors=1)),
+ ('failed_upload', dict(migrated=0, source=2, missing=2, errors=1))
+]
+
+
+@pytest.mark.parametrize('test, assertions', mirgation_test_specs)
@pytest.mark.timeout(10)
-def test_migrate(migrate_infra, upload, assertions):
- upload_path = os.path.join('tests', 'data', 'migration', upload, 'upload')
- reports = list(migrate_infra.migrate(upload_path))
+def test_migrate(migrate_infra, test, assertions, caplog):
+ uploads_path = os.path.join('tests', 'data', 'migration', test)
+ reports = list(migrate_infra.migrate(
+ *[os.path.join(uploads_path, dir) for dir in os.listdir(uploads_path)]))
+
assert len(reports) == 1
report = reports[0]
- assert report['status'] == SUCCESS
- assert report['total_calcs'] == 2
- assert report['total_source_calcs'] == 2
- assert report['migrated_calcs'] == 2
- assert report['calcs_with_diffs'] == 0
- assert report['new_calcs'] == 0
- assert report['missing_calcs'] == 0
+ assert report['total_calcs'] == assertions.get('migrated', 0) + assertions.get('new', 0) + assertions.get('failed', 0)
+
+ assert report['total_source_calcs'] == assertions.get('source', 0)
+ assert report['migrated_calcs'] == assertions.get('migrated', 0)
+ assert report['calcs_with_diffs'] == assertions.get('diffs', 0)
+ assert report['new_calcs'] == assertions.get('new', 0)
+ assert report['missing_calcs'] == assertions.get('missing', 0)
+
+ errors = 0
+ for record in caplog.get_records(when='call'):
+ if record.levelname in ['ERROR', 'CRITICAL']:
+ record_data = json.loads(record.getMessage())
+ if 'source_upload_id' in record_data:
+ errors += 1
+
+ assert errors == assertions.get('errors', 0)