Skip to content
GitLab
Commit 8677c7fb authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Completed migration tests.

parent 5f4318fa
Hide whitespace changes
Inline Side-by-side
nomad/api/app.py
View file @ 8677c7fb
......@@ -66,6 +66,8 @@ api = Api(
@api.errorhandler
def handle(error: Exception):
status_code = getattr(error, 'code', 500)
if not isinstance(status_code, int):
status_code = 500
name = getattr(error, 'name', 'Internal Server Error')
description = getattr(error, 'description', 'No description available')
data = dict(
......
nomad/api/upload.py
View file @ 8677c7fb
......@@ -172,24 +172,13 @@ class UploadListResource(Resource):
if upload.name is None or upload.name is '':
upload.name = file.filename
upload_files = ArchiveBasedStagingUploadFiles(
upload.upload_id, create=True, local_path=local_path,
file_name='.upload.%s' % os.path.splitext(file.filename)[1])
upload_files = ArchiveBasedStagingUploadFiles(upload.upload_id, create=True)
file.save(upload_files.upload_file_os_path)
else:
# simple streaming data in HTTP body, e.g. with curl "url" -T local_file
file_name = '.upload'
try:
ext = os.path.splitext(upload.name)[1]
if ext is not None:
file_name += '.' + ext
except Exception:
pass
upload_files = ArchiveBasedStagingUploadFiles(
upload.upload_id, create=True, local_path=local_path,
file_name='.upload')
upload_files = ArchiveBasedStagingUploadFiles(upload.upload_id, create=True)
try:
with open(upload_files.upload_file_os_path, 'wb') as f:
......
nomad/migration.py
View file @ 8677c7fb
......@@ -91,7 +91,11 @@ class SourceCalc(Document):
total = source_query.count()
while True:
calcs = source_query.filter(Calc.coe_calc_id > start_pid).order_by(Calc.coe_calc_id).limit(per_query)
calcs = source_query \
.filter(Calc.coe_calc_id > start_pid) \
.order_by(Calc.coe_calc_id) \
.limit(per_query)
source_calcs = []
for calc in calcs:
if calc.calc_metadata is None or calc.calc_metadata.filenames is None:
......@@ -269,11 +273,12 @@ class NomadCOEMigration:
source_upload_id = os.path.split(os.path.split(upload_path)[0])[1]
upload_name = os.path.basename(upload_path)
else:
potential_upload_archive = os.path.join(upload_path, NomadCOEMigration.archive_filename)
potential_upload_archive = os.path.join(
upload_path, NomadCOEMigration.archive_filename)
if os.path.isfile(potential_upload_archive):
upload_archive_f = open(potential_upload_archive, 'rb')
source_upload_id = os.path.split(os.path.split(potential_upload_archive)[0])[1]
upload_name = '%s.tar.gz' % source_upload_id
upload_name = os.path.basename(potential_upload_archive)
else:
source_upload_id = os.path.split(upload_path)[1]
zip_file = zipstream.ZipFile()
......@@ -289,22 +294,13 @@ class NomadCOEMigration:
upload_name = '%s.zip' % source_upload_id
# upload and process the upload file
upload = self.client.uploads.upload(file=upload_archive_f, name=upload_name).response().result
upload = self.client.uploads.upload(
file=upload_archive_f, name=upload_name).response().result
upload_archive_f.close()
upload_logger = self.logger.bind(
source_upload_id=source_upload_id, upload_id=upload.upload_id)
while upload.tasks_running:
upload = self.client.uploads.get_upload(upload_id=upload.upload_id).response().result
time.sleep(0.1)
if upload.tasks_status == FAILURE:
error = 'failed to process upload'
upload_logger.logger.error(error, process_errors=upload.errors)
return dict(error=error, status=FAILURE)
# grab source metadata
upload_metadata_calcs = list()
metadata_dict = dict()
......@@ -313,20 +309,34 @@ class NomadCOEMigration:
source_metadata = CalcWithMetadata(upload_id=upload.upload_id, **source_calc.metadata)
source_metadata.mainfile = source_calc.mainfile
source_metadata.pid = source_calc.pid
source_metadata.__migrated = None
source_metadata.__migrated = False
upload_metadata_calcs.append(source_metadata)
metadata_dict[source_calc.mainfile] = source_metadata
# verify upload
report = utils.POPO()
report.total_calcs = upload.calcs.pagination.total
report.total_source_calcs = len(metadata_dict)
report.unprocessed_calcs = 0
report.failed_calcs = 0
report.migrated_calcs = 0
report.calcs_with_diffs = 0
report.new_calcs = 0
report.missing_calcs = 0
# wait for complete upload
while upload.tasks_running:
upload = self.client.uploads.get_upload(upload_id=upload.upload_id).response().result
time.sleep(0.1)
if upload.tasks_status == FAILURE:
error = 'failed to process upload'
report.missing_calcs = report.total_source_calcs
report.total_calcs = 0
upload_logger.error(error, process_errors=upload.errors)
yield report
continue
else:
report.total_calcs = upload.calcs.pagination.total
# verify upload
for page in range(1, math.ceil(report.total_calcs / 100) + 1):
upload = self.client.uploads.get_upload(
upload_id=upload.upload_id, per_page=100, page=page,
......@@ -351,22 +361,17 @@ class NomadCOEMigration:
upload.upload_id, calc_proc.calc_id, source_calc, calc_logger):
report.calcs_with_diffs += 1
else:
report.unprocessed_calcs += 1
calc_logger.info(
'could not process a calc%s.' %
', that is in source' if source_calc is not None else '')
if source_calc is not None:
source_calc.__migrated = False
report.failed_calcs += 1
calc_logger.error(
'could not process a calc', process_errors=calc_proc.errors)
continue
for source_calc in upload_metadata_calcs:
if source_calc.__migrated is None:
if source_calc.__migrated is False:
report.missing_calcs += 1
upload_logger.info('no match for source calc', mainfile=source_calc.mainfile)
elif source_calc.__migrated is False:
upload_logger.info('source calc not processed', mainfile=source_calc.mainfile)
upload_logger.info(
'no match or processed calc for source calc',
mainfile=source_calc.mainfile)
# commit upload
admin_keys = ['upload_time, uploader, pid']
......@@ -388,7 +393,7 @@ class NomadCOEMigration:
transform(calc) for calc in upload_metadata['calculations']
if calc.__migrated]
if report.total_calcs > report.unprocessed_calcs:
if report.total_calcs > report.failed_calcs:
upload = self.client.uploads.exec_upload_command(
upload_id=upload.upload_id,
payload=dict(command='commit', metadata=upload_metadata)
......@@ -405,7 +410,6 @@ class NomadCOEMigration:
# report
upload_logger.info('migrated upload', **report)
report.update(status=SUCCESS)
yield report
def index(self, *args, **kwargs):
......
nomad/processing/data.py
View file @ 8677c7fb
......@@ -495,4 +495,4 @@ class Upload(Chord, datamodel.Upload):
@property
def calcs(self):
return Calc.objects(upload_id=self.upload_id)
return Calc.objects(upload_id=self.upload_id, tasks_status=SUCCESS)
tests/bravado_flaks.py
View file @ 8677c7fb
......@@ -92,9 +92,8 @@ class FlaskTestFutureAdapter:
if len(files) > 1:
raise NotImplementedError
if len(files) == 1:
data = dict() if data is None else data
_, (_, f) = files[0]
data.update(file=(f, 'file'))
data = f
return function(
url, headers=self._request_params.get('headers'), data=data)
......
tests/data/migration/failed_calc/upload/1/template.json 0 → 100644
View file @ 8677c7fb
{
"section_run": [
{
"_name": "section_run",
"_gIndex": 0,
"program_name": "VASP",
"program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
"program_basis_set_type": "plane waves",
"section_method": [
{
"_name": "section_method",
"_gIndex": 0,
"electronic_structure_method": "DFT",
"section_XC_functionals": [
{
"_name": "section_XC_functionals",
"_gIndex": 0,
"XC_functional_name": "GGA_X_PBE"
}
]
}
],
"section_system": [
{
"_name": "section_system",
"_gIndex": 0,
"simulation_cell": [
[
5.76372622e-10,
0.0,
0.0
],
[
0.0,
5.76372622e-10,
0.0
],
[
0.0,
0.0,
4.0755698899999997e-10
]
],
"configuration_periodic_dimensions": [
true,
true,
true
],
"atom_positions": [
[
2.88186311e-10,
0.0,
2.0377849449999999e-10
],
[
0.0,
2.88186311e-10,
2.0377849449999999e-10
],
[
0.0,
0.0,
0.0
],
[
2.88186311e-10,
2.88186311e-10,
0.0
]
],
"atom_labels": [
"Br",
"K",
"Si",
"Si"
]
}
],
"section_single_configuration_calculation": [
{
"_name": "section_single_configuration_calculation",
"_gIndex": 0,
"single_configuration_calculation_to_system_ref": 0,
"single_configuration_to_calculation_method_ref": 0,
"energy_free": -1.5936767191492225e-18,
"energy_total": -1.5935696296699573e-18,
"energy_total_T0": -3.2126683561907e-22
}
],
"section_sampling_method": [
{
"_name": "section_sampling_method",
"_gIndex": 0,
"sampling_method": "geometry_optimization"
}
],
"section_frame_sequence": [
{
"_name": "section_frame_sequence",
"_gIndex": 0,
"frame_sequence_to_sampling_ref": 0,
"frame_sequence_local_frames_ref": [
0
]
}
]
}
]
}
\ No newline at end of file
tests/data/migration/failed_calc/upload/2/template.json 0 → 100644
View file @ 8677c7fb
{
"section_run": [
{
not parsable
\ No newline at end of file
tests/data/migration/missing_calc/upload/1/template.json 0 → 100644
View file @ 8677c7fb
{
"section_run": [
{
"_name": "section_run",
"_gIndex": 0,
"program_name": "VASP",
"program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
"program_basis_set_type": "plane waves",
"section_method": [
{
"_name": "section_method",
"_gIndex": 0,
"electronic_structure_method": "DFT",
"section_XC_functionals": [
{
"_name": "section_XC_functionals",
"_gIndex": 0,
"XC_functional_name": "GGA_X_PBE"
}
]
}
],
"section_system": [
{
"_name": "section_system",
"_gIndex": 0,
"simulation_cell": [
[
5.76372622e-10,
0.0,
0.0
],
[
0.0,
5.76372622e-10,
0.0
],
[
0.0,
0.0,
4.0755698899999997e-10
]
],
"configuration_periodic_dimensions": [
true,
true,
true
],
"atom_positions": [
[
2.88186311e-10,
0.0,
2.0377849449999999e-10
],
[
0.0,
2.88186311e-10,
2.0377849449999999e-10
],
[
0.0,
0.0,
0.0
],
[
2.88186311e-10,
2.88186311e-10,
0.0
]
],
"atom_labels": [
"Br",
"K",
"Si",
"Si"
]
}
],
"section_single_configuration_calculation": [
{
"_name": "section_single_configuration_calculation",
"_gIndex": 0,
"single_configuration_calculation_to_system_ref": 0,
"single_configuration_to_calculation_method_ref": 0,
"energy_free": -1.5936767191492225e-18,
"energy_total": -1.5935696296699573e-18,
"energy_total_T0": -3.2126683561907e-22
}
],
"section_sampling_method": [
{
"_name": "section_sampling_method",
"_gIndex": 0,
"sampling_method": "geometry_optimization"
}
],
"section_frame_sequence": [
{
"_name": "section_frame_sequence",
"_gIndex": 0,
"frame_sequence_to_sampling_ref": 0,
"frame_sequence_local_frames_ref": [
0
]
}
]
}
]
}
\ No newline at end of file
tests/data/migration/missmatch/upload/1/template.json 0 → 100644
View file @ 8677c7fb
{
"section_run": [
{
"_name": "section_run",
"_gIndex": 0,
"program_name": "VASP",
"program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
"program_basis_set_type": "plane waves",
"section_method": [
{
"_name": "section_method",
"_gIndex": 0,
"electronic_structure_method": "DFT",
"section_XC_functionals": [
{
"_name": "section_XC_functionals",
"_gIndex": 0,
"XC_functional_name": "GGA_X_PBE"
}
]
}
],
"section_system": [
{
"_name": "section_system",
"_gIndex": 0,
"simulation_cell": [
[
5.76372622e-10,
0.0,
0.0
],
[
0.0,
5.76372622e-10,
0.0
],
[
0.0,
0.0,
4.0755698899999997e-10
]
],
"configuration_periodic_dimensions": [
true,
true,
true
],
"atom_positions": [
[
2.88186311e-10,
0.0,
2.0377849449999999e-10
],
[
0.0,
2.88186311e-10,
2.0377849449999999e-10
],
[
0.0,
0.0,
0.0
],
[
2.88186311e-10,
2.88186311e-10,
0.0
]
],
"atom_labels": [
"Br",
"K",
"Si",
"Si"
]
}
],
"section_single_configuration_calculation": [
{
"_name": "section_single_configuration_calculation",
"_gIndex": 0,
"single_configuration_calculation_to_system_ref": 0,
"single_configuration_to_calculation_method_ref": 0,
"energy_free": -1.5936767191492225e-18,
"energy_total": -1.5935696296699573e-18,
"energy_total_T0": -3.2126683561907e-22
}
],
"section_sampling_method": [
{
"_name": "section_sampling_method",
"_gIndex": 0,
"sampling_method": "geometry_optimization"
}
],
"section_frame_sequence": [
{
"_name": "section_frame_sequence",
"_gIndex": 0,
"frame_sequence_to_sampling_ref": 0,
"frame_sequence_local_frames_ref": [
0
]
}
]
}
]
}
\ No newline at end of file
tests/data/migration/missmatch/upload/2/template.json 0 → 100644
View file @ 8677c7fb
{
"section_run": [
{
"_name": "section_run",
"_gIndex": 0,
"program_name": "VASP",
"program_version": "4.6.35 3Apr08 complex parallel LinuxIFC",
"program_basis_set_type": "plane waves",
"section_method": [
{
"_name": "section_method",
"_gIndex": 0,
"electronic_structure_method": "DFT",
"section_XC_functionals": [
{
"_name": "section_XC_functionals",
"_gIndex": 0,
"XC_functional_name": "missmatch"
}
]
}
],
"section_system": [
{
"_name": "section_system",
"_gIndex": 0,
"simulation_cell": [
[
5.76372622e-10,
0.0,
0.0
],
[
0.0,
5.76372622e-10,
0.0
],
[
0.0,
0.0,
4.0755698899999997e-10
]
],
"configuration_periodic_dimensions": [
true,
true,
true
],
"atom_positions": [
[
2.88186311e-10,
0.0,
2.0377849449999999e-10
],
[
0.0,
2.88186311e-10,
2.0377849449999999e-10