Commit 422ed1e2 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.10.7' into 'master'

Merge for release

Closes #601, #602, #606, #605, and #603

See merge request !384
parents fc8f994b d2372917
Pipeline #112017 canceled with stages
in 12 minutes and 42 seconds
......@@ -204,3 +204,6 @@
[submodule "dependencies/parsers/lobster"]
path = dependencies/parsers/lobster
url = https://github.com/ondracka/nomad_parser_lobster.git
[submodule "dependencies/parsers/openmx"]
path = dependencies/parsers/openmx
url = https://github.com/ondracka/nomad-parser-openmx.git
......@@ -46,6 +46,9 @@ contributing, and API reference.
Omitted versions are plain bugfix releases with only minor changes and fixes.
### v0.10.7
- adding OpenMX parser
### v0.10.6
- support for NOMAD fields in optimade
......
Subproject commit 0c00a130ae16c03a8527217aa2ecc46d11c7ddb7
{
"name": "nomad-fair-gui",
"version": "0.10.5",
"version": "0.10.7",
"commit": "e98694e",
"private": true,
"workspaces": [
......
......@@ -9,7 +9,7 @@ window.nomadEnv = {
'matomoUrl': 'https://nomad-lab.eu/fairdi/stat',
'matomoSiteId': '2',
'version': {
'label': '0.10.5',
'label': '0.10.7',
'isBeta': false,
'isTest': true,
'usesBetaData': true,
......
......@@ -8,7 +8,7 @@ global.nomadEnv = {
'matomoUrl': 'https://nomad-lab.eu/fairdi/stat',
'matomoSiteId': '2',
'version': {
'label': '0.10.5',
'label': '0.10.7',
'isBeta': false,
'isTest': true,
'usesBetaData': true,
......
......@@ -174,13 +174,13 @@ class MirrorFilesResource(Resource):
return send_file(
open(fileobj.os_path, 'rb'),
mimetype='application/zip',
mimetype='application/binary',
as_attachment=True,
cache_timeout=0,
attachment_filename=fileobj.os_path)
except KeyError:
abort(404, message='Upload %d does not exist' % upload_id)
abort(404, message='Upload %s does not exist' % upload_id)
@ns.route('/users')
......
......@@ -663,6 +663,24 @@ class EditRepoCalcsResource(Resource):
else:
mongo_value = action_value
# verify if the edit action creates consistent shared_with and with_embargo for
# whole uploads
if action_quantity_name == 'shared_with' or action_quantity_name == 'with_embargo':
search_request = search.SearchRequest()
apply_search_parameters(search_request, parsed_query)
search_request.quantity('upload_id')
uploads = search_request.execute()['quantities']['upload_id']['values']
for upload_id, upload_data in uploads.items():
search_request = search.SearchRequest().search_parameters(upload_id=upload_id)
entries = search_request.execute()['total']
if entries != upload_data['total']:
action['success'] = False
action['message'] = json_data.get('message', '') + (
'Edit would create an upload with inconsistent shared_with or with_embargo. '
'You can only set those for all entries of an upload.')
has_error = True
continue
if len(quantity.shape) == 0:
mongo_update[mongo_key] = mongo_value
else:
......@@ -715,7 +733,8 @@ class EditRepoCalcsResource(Resource):
if lift_embargo:
for upload_id in upload_ids:
upload = proc.Upload.get(upload_id)
upload.re_pack()
if upload.published:
upload.re_pack()
# remove potentially empty old datasets
if removed_datasets is not None:
......
......@@ -197,7 +197,7 @@ async def post_datasets(
'''
now = datetime.now()
dataset_type = create.dataset_type if create.dataset_type is not None else 'owned'
dataset_type = create.dataset_type if create.dataset_type is not None else DatasetType.owned
# check if name already exists
existing_dataset = DatasetDefinitionCls.m_def.a_mongo.objects(
......@@ -217,22 +217,28 @@ async def post_datasets(
dataset_type=dataset_type)
dataset.a_mongo.create()
# add dataset to entries in mongo and elastic
# TODO this should be part of a new edit API
if create.entries is not None:
es_query = cast(Query, {'calc_id': Any_(any=create.entries)})
mongo_query = {'_id': {'$in': create.entries}}
empty = len(create.entries) == 0
elif create.query is not None:
es_query = create.query
entries = _do_exaustive_search(
owner=Owner.public, query=create.query, user=user,
include=['calc_id'])
entry_ids = [entry['calc_id'] for entry in entries]
mongo_query = {'_id': {'$in': entry_ids}}
empty = len(entry_ids) == 0
else:
if dataset_type != DatasetType.owned:
dataset.query = create.query
dataset.entrys = create.entries
empty = True
else:
# add dataset to entries in mongo and elastic
# TODO this should be part of a new edit API
if create.entries is not None:
es_query = cast(Query, {'calc_id': Any_(any=create.entries)})
elif create.query is not None:
es_query = create.query
else:
es_query = None
if es_query is None:
empty = True
else:
entries = _do_exaustive_search(
owner=Owner.user, query=es_query, user=user, include=['calc_id'])
entry_ids = [entry['calc_id'] for entry in entries]
mongo_query = {'_id': {'$in': entry_ids}}
empty = len(entry_ids) == 0
if not empty:
processing.Calc._get_collection().update_many(
......
......@@ -284,7 +284,7 @@ datacite = NomadConfig(
)
meta = NomadConfig(
version='0.10.6',
version='0.10.7',
commit=gitinfo.commit,
release='devel',
deployment='standard',
......
......@@ -231,6 +231,10 @@ class Dataset(metainfo.MSection):
type=metainfo.MEnum('owned', 'foreign'),
a_mongo=Mongo(index=True),
a_search=Search())
query = metainfo.Quantity(
type=metainfo.JSON, a_mongo=Mongo())
entries = metainfo.Quantity(
type=str, shape=['*'], a_mongo=Mongo())
class DatasetReference(metainfo.Reference):
......
......@@ -63,6 +63,7 @@ import hashlib
import io
import pickle
import json
from more_itertools import peekable
from nomad import config, utils, datamodel
from nomad.archive import write_archive, read_archive, ArchiveReader
......@@ -461,8 +462,10 @@ class StagingUploadFiles(UploadFiles):
mode='w')
def write_msgfile(access: str, size: int, data: Iterable[Tuple[str, Any]]):
file_object = PublicUploadFiles._create_msg_file_object(target_dir, access)
write_archive(file_object.os_path, size, data)
data = peekable(data)
if data:
file_object = PublicUploadFiles._create_msg_file_object(target_dir, access)
write_archive(file_object.os_path, size, data)
# zip archives
if not skip_archive:
......@@ -503,13 +506,13 @@ class StagingUploadFiles(UploadFiles):
return restricted, public
def _pack_raw_files(self, entries: Iterable[datamodel.EntryMetadata], create_zipfile):
raw_public_zip = create_zipfile('public')
raw_restricted_zip = create_zipfile('restricted')
raw_public_zip, raw_restricted_zip = None, None
try:
# 1. add all public raw files
# 1.1 collect all public mainfiles and aux files
public_files: Dict[str, str] = {}
restricted_files: Dict[str, str] = {}
for calc in entries:
if not calc.with_embargo:
mainfile = calc.mainfile
......@@ -519,28 +522,51 @@ class StagingUploadFiles(UploadFiles):
for filepath in self.calc_files(mainfile, with_cutoff=False):
if not always_restricted(filepath):
public_files[filepath] = None
# 1.2 remove the non public mainfiles that have been added as auxfiles of public mainfiles
for calc in entries:
if calc.with_embargo:
mainfile = calc.mainfile
assert mainfile is not None
restricted_files[mainfile] = None
if mainfile in public_files:
del(public_files[mainfile])
# 1.3 zip all remaining public
else:
for filepath in self.calc_files(mainfile, with_cutoff=False):
restricted_files[filepath] = None
# 1.3 zip all public
for filepath in public_files.keys():
if raw_public_zip is None:
raw_public_zip = create_zipfile('public')
raw_public_zip.write(self._raw_dir.join_file(filepath).os_path, filepath)
# 2. everything else becomes restricted
# 1.4 zip all restructed
for filepath in restricted_files.keys():
if raw_restricted_zip is None:
raw_restricted_zip = create_zipfile('restricted')
raw_restricted_zip.write(self._raw_dir.join_file(filepath).os_path, filepath)
# 2. everything else becomes restricted (or public if everything else was public)
if raw_restricted_zip is None:
raw_zip = raw_public_zip
else:
if raw_restricted_zip is None:
raw_restricted_zip = create_zipfile('restricted')
raw_zip = raw_restricted_zip
for filepath in self.raw_file_manifest():
if filepath not in public_files:
raw_restricted_zip.write(self._raw_dir.join_file(filepath).os_path, filepath)
if filepath not in public_files and filepath not in restricted_files:
raw_zip.write(self._raw_dir.join_file(filepath).os_path, filepath)
except Exception as e:
self.logger.error('exception during packing raw files', exc_info=e)
finally:
raw_restricted_zip.close()
raw_public_zip.close()
if raw_restricted_zip is not None:
raw_restricted_zip.close()
if raw_public_zip is not None:
raw_public_zip.close()
def raw_file_manifest(self, path_prefix: str = None) -> Generator[str, None, None]:
upload_prefix_len = len(self._raw_dir.os_path) + 1
......@@ -698,10 +724,14 @@ class PublicUploadFilesBasedStagingUploadFiles(StagingUploadFiles):
super().add_rawfiles(raw_file_zip.os_path, force_archive=True)
if include_archive:
with self.public_upload_files._open_msg_file(access) as archive:
for calc_id, data in archive.items():
calc_id = calc_id.strip()
self.write_archive(calc_id, data.to_dict())
try:
with self.public_upload_files._open_msg_file(access) as archive:
for calc_id, data in archive.items():
calc_id = calc_id.strip()
self.write_archive(calc_id, data.to_dict())
except FileNotFoundError:
# ignore missing archive file and assume equivalent to empty archive file
pass
def add_rawfiles(self, *args, **kwargs) -> None:
assert False, 'do not add_rawfiles to a %s' % self.__class__.__name__
......@@ -939,8 +969,10 @@ class PublicUploadFiles(UploadFiles):
return zipfile.ZipFile(file.os_path, mode='w')
def write_msgfile(access: str, size: int, data: Iterable[Tuple[str, Any]]):
file = self._msg_file_object(access, suffix='repacked')
write_archive(file.os_path, size, data)
data = peekable(data)
if data:
file = self._msg_file_object(access, suffix='repacked')
write_archive(file.os_path, size, data)
# perform the repacking
try:
......@@ -954,6 +986,9 @@ class PublicUploadFiles(UploadFiles):
# replace the original files with the repacked ones
for repacked_file, public_file in files:
shutil.move(
repacked_file.os_path,
public_file.os_path)
if repacked_file.exists():
shutil.move(
repacked_file.os_path,
public_file.os_path)
elif public_file.exists():
public_file.delete()
......@@ -20,7 +20,7 @@ from flask_restplus import fields
from nomad.app.flask.common import RFC3339DateTime
from .metainfo import Section, Quantity, Datetime, Capitalized, MEnum
from .metainfo import Section, Quantity, Datetime, Capitalized, MEnum, JSON
def field(quantity: Quantity):
......@@ -38,6 +38,8 @@ def field(quantity: Quantity):
field = RFC3339DateTime
elif isinstance(quantity.type, MEnum):
field = fields.String
elif quantity.type == JSON:
field = fields.Arbitrary
else:
raise NotImplementedError
......
......@@ -37,7 +37,7 @@ from typing import Any, Dict, List
from .metainfo import (
DefinitionAnnotation, SectionAnnotation, Annotation, MSection, Datetime, Quantity,
MEnum)
MEnum, JSON)
class Mongo(DefinitionAnnotation):
......@@ -103,6 +103,8 @@ class MongoDocument(SectionAnnotation):
field = me.DateTimeField
elif isinstance(quantity.type, MEnum):
field = me.StringField
elif quantity.type == JSON:
field = me.DictField
else:
raise NotImplementedError
......
......@@ -36,7 +36,7 @@ from typing import cast
from pydantic import create_model, Field, BaseConfig
from datetime import datetime
from .metainfo import DefinitionAnnotation, Definition, Section, Quantity, Datetime, MEnum, Capitalized
from .metainfo import DefinitionAnnotation, Definition, Section, Quantity, Datetime, MEnum, Capitalized, JSON
class _OrmConfig(BaseConfig):
......@@ -71,6 +71,8 @@ class PydanticModel(DefinitionAnnotation):
pydantic_type = str
elif quantity.type == Capitalized:
pydantic_type = str
elif quantity.type == JSON:
pydantic_type = dict
else:
pydantic_type = quantity.type
......
......@@ -72,6 +72,7 @@ from libatomsparser import LibAtomsParser
from atkparser import ATKParser
from qboxparser import QboxParser
from openkimparser import OpenKIMParser
from openmxparser import OpenmxParser
try:
# these packages are not available without parsing extra, which is ok, if the
......@@ -212,6 +213,7 @@ parsers = [
AsapParser(),
FploParser(),
MopacParser(),
OpenmxParser(),
ArchiveParser()
]
......
......@@ -615,7 +615,7 @@ class Calc(Proc):
logger.info('Apply user metadata from nomad.yaml/json file')
for key, val in metadata.items():
if key == 'entries':
if key in ['entries', 'skip_matching']:
continue
definition = _editable_metadata.get(key, None)
......@@ -1247,10 +1247,19 @@ class Upload(Proc):
Returns:
Tuples of mainfile, filename, and parsers
'''
metadata = self.metadata_file_cached(
os.path.join(self.upload_files.os_path, 'raw', config.metadata_file_name))
skip_matching = metadata.get('skip_matching', False)
entries_metadata = metadata.get('entries', {})
directories_with_match: Dict[str, str] = dict()
upload_files = self.upload_files.to_staging_upload_files()
for filename in upload_files.raw_file_manifest():
self._preprocess_files(filename)
if skip_matching and filename not in entries_metadata:
continue
try:
parser = match_parser(upload_files.raw_file_object(filename).os_path)
if parser is not None:
......
apiVersion: v1
appVersion: "0.10.6"
appVersion: "0.10.7"
description: A Helm chart for Kubernetes that only runs nomad services and uses externally hosted databases.
name: nomad
version: 0.10.6
version: 0.10.7
......@@ -62,3 +62,5 @@ volumes:
staging: /nomad/fairdi/prod/fs/staging
tmp: /nomad/fairdi/prod/fs/tmp
nomad: /nomad
reprocess_match: true
......@@ -18,6 +18,7 @@ data:
{{ if .Values.meta.deployment }}
deployment: "{{ .Values.meta.deployment }}"
{{ end }}
reprocess_match: {{ .Values.reprocess_match }}
reprocess_unmatched: {{ .Values.reprocess_unmatched }}
reprocess_rematch: {{ .Values.reprocess_rematch }}
process_reuse_parser: {{ .Values.process_reuse_parser }}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment