Commit 9b950a50 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'experimental' of gitlab.mpcdf.mpg.de:nomad-lab/nomad-FAIR into experimental

parents d48f1f29 d6254cc6
Pipeline #47361 canceled with stages
in 14 seconds
......@@ -114,7 +114,7 @@ class PathObject:
@property
def size(self) -> int:
""" Returns the os determined file size. """
""" The os determined file size. """
return os.stat(self.os_path).st_size
def __repr__(self) -> str:
......@@ -176,7 +176,10 @@ class UploadFiles(DirectoryObject, metaclass=ABCMeta):
self.upload_id = upload_id
self._is_authorized = is_authorized
self._user_metadata_file = self.join_file('user_metadata.pickle')
@property
def _user_metadata_file(self):
return self.join_file('user_metadata.pickle')
@property
def user_metadata(self) -> dict:
......@@ -248,14 +251,21 @@ class UploadFiles(DirectoryObject, metaclass=ABCMeta):
class StagingUploadFiles(UploadFiles):
def __init__(self, *args, **kwargs) -> None:
super().__init__(config.fs.staging, *args, **kwargs)
def __init__(
self, upload_id: str, is_authorized: Callable[[], bool] = lambda: False,
create: bool = False) -> None:
super().__init__(config.fs.staging, upload_id, is_authorized, create)
self._raw_dir = self.join_dir('raw')
self._archive_dir = self.join_dir('archive')
self._frozen_file = self.join_file('.frozen')
self._size = 0
self._shared = DirectoryObject(config.fs.public, upload_id, create=create)
@property
def _user_metadata_file(self):
return self._shared.join_file('user_metadata.pickle')
@property
def size(self) -> int:
......@@ -368,10 +378,11 @@ class StagingUploadFiles(UploadFiles):
assert target_dir.exists()
# copy user metadata
if self._user_metadata_file.exists():
target_metadata_file = target_dir.join_file(user_metadata_filename)
if self._user_metadata_file.exists() and not target_metadata_file.exists():
shutil.copyfile(
self._user_metadata_file.os_path,
target_dir.join_file(user_metadata_filename).os_path)
target_metadata_file.os_path)
def create_zipfile(kind: str, prefix: str, ext: str) -> ZipFile:
file = target_dir.join_file('%s-%s.%s.zip' % (kind, prefix, ext))
......@@ -506,6 +517,11 @@ class StagingUploadFiles(UploadFiles):
return utils.make_websave(hash)
def delete(self) -> None:
super().delete()
if self._shared.exists():
self._shared.delete()
class ArchiveBasedStagingUploadFiles(StagingUploadFiles):
"""
......
......@@ -423,24 +423,14 @@ class SourceCalc(Document):
if package is None:
data.uploads_with_no_package.append(source_upload)
else:
source_uploads.append(source_upload)
calcs = SourceCalc.objects(upload=source_upload).count()
packages = Package.objects(upload_id=source_upload).count()
source_uploads.append(dict(
id=source_upload, packages=packages, calcs=calcs,
path=package.upload_path))
source_uploads = sorted(source_uploads, key=lambda k: k['calcs'])
data.source_uploads = source_uploads
data.step = 2
if data.step < 3:
source_uploads = []
for source_upload in data.source_uploads:
count = SourceCalc.objects(upload=source_upload).count()
source_uploads.append(utils.POPO(id=source_upload, calcs=count))
data.source_uploads = sorted(source_uploads, key=lambda k: k['calcs'])
data.step = 3
if data.step < 4:
source_uploads = []
for source_upload in data.source_uploads:
count = Package.objects(upload_id=source_upload.get('id')).count()
source_upload['packages'] = count
data.step = 4
finally:
with open(tmp_data_path, 'wt') as f:
json.dump(data, f)
......@@ -853,15 +843,31 @@ class NomadCOEMigration:
report = Report()
report.total_packages += 1
# check if the package is already uploaded
upload = None
try:
uploads = self.call_api('uploads.get_uploads')
for a_upload in uploads:
if a_upload.name == package_id and len(a_upload.errors) == 0:
assert upload is None, 'duplicate upload name'
upload = a_upload
except Exception as e:
self.logger.error('could verify if upload already exists', exc_info=e)
report.failed_packages += 1
return report
# upload and process the upload file
with utils.timer(logger, 'upload completed'):
try:
upload = self.call_api(
'uploads.upload', name=package_id, local_path=package.package_path)
except Exception as e:
self.logger.error('could not upload package', exc_info=e)
report.failed_packages += 1
return report
if upload is None:
with utils.timer(logger, 'upload completed'):
try:
upload = self.call_api(
'uploads.upload', name=package_id, local_path=package.package_path)
except Exception as e:
self.logger.error('could not upload package', exc_info=e)
report.failed_packages += 1
return report
else:
self.logger.info('package was already uploaded')
logger = logger.bind(
source_upload_id=source_upload_id, upload_id=upload.upload_id)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment