Commit 335008d7 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.10.4' into 'master'

Merge for release

See merge request !357
parents 0bde4980 6919ccf6
Pipeline #103569 passed with stage
in 4 minutes and 55 seconds
......@@ -353,8 +353,8 @@ class UploadPage extends React.Component {
\`\`\`
### Form data vs. streaming
NOMAD accepts stream data (\`-T <local_file>\`) (like in the
examples above) or multi-part form data (\`-X PUT -f file=@<local_file>\`):
NOMAD accepts stream data \`-T <local_file>\` (like in the
examples above) or multi-part form data \`-X PUT -F file=@<local_file>\`:
\`\`\`
${uploadCommand.upload_command_form}
\`\`\`
......@@ -363,8 +363,8 @@ class UploadPage extends React.Component {
more information (e.g. the file name) to our servers (see below).
#### Upload names
With multi-part form data (\`-X PUT -f file=@<local_file>\`), your upload will
be named after the file by default. With stream data (\`-T <local_file>\`)
With multi-part form data \`-X PUT -F file=@<local_file>\`, your upload will
be named after the file by default. With stream data \`-T <local_file>\`
there will be no default name. To set a custom name, you can use the URL
parameter \`name\`:
\`\`\`
......
......@@ -8,7 +8,7 @@ global.nomadEnv = {
'matomoUrl': 'https://nomad-lab.eu/fairdi/stat',
'matomoSiteId': '2',
'version': {
'label': '0.10.3',
'label': '0.10.4',
'isBeta': false,
'isTest': true,
'usesBetaData': true,
......
......@@ -24,9 +24,7 @@ The archive API of the nomad@FAIRDI APIs. This API is about serving processed
from flask_restplus import abort, Resource
import importlib
from nomad.metainfo.legacy import python_package_mapping, LegacyMetainfoEnvironment
from nomad.metainfo import Package
from nomad.parsing.parsers import parsers
from .api import api
......@@ -44,10 +42,7 @@ class AllMetainfoResource(Resource):
'''
Returns all metainfo packages.
'''
# Ensure all metainfo is loaded
for parser in parsers:
_ = parser.metainfo_env
from nomad.parsing.parsers import parsers # pylint: disable=unused-import
return {
key: value.m_to_dict()
for key, value in Package.registry.items()}
......@@ -60,25 +55,16 @@ class MetainfoResource(Resource):
@api.response(200, 'Metainfo send')
def get(self, metainfo_package_name):
'''
Get a JSON representation of the NOMAD Metainfo.
You can get the metainfo for 'common', and parser/code metainfo packages.
Parser/code packages constain the necessary definitions that the respective
parser/code might use. 'Common' contains all non specific general definitions.
Other required packages might also be returned, e.g. a parser might organize its
definitions in multiple packages.
Get a JSON representation of a NOMAD Metainfo package. The package name is
the qualified Python name of the respective module that contains the definitions.
Examples are `nomad.datamodel.metainfo.common_dft` or `vaspparser.metainfo`.
If the desired package depends on other packages, these will also be contain in
the results.
'''
package = metainfo_package_name
if package.endswith('.json'):
package = package[:-5]
try:
try:
python_module = importlib.import_module(package)
except ImportError:
python_package_name, _ = python_package_mapping(package)
python_module = importlib.import_module(python_package_name)
python_module = importlib.import_module(package)
metainfo_package = getattr(python_module, 'm_package')
except (ImportError, KeyError, FileNotFoundError, AttributeError):
abort(404, message='Metainfo package %s does not exist.' % package)
......@@ -89,29 +75,3 @@ class MetainfoResource(Resource):
result[dependency.name] = dependency.m_to_dict()
return result
@ns.route('/legacy/<string:metainfo_package_name>')
class LegacyMetainfoResource(Resource):
@api.doc('get_legacy_metainfo')
@api.response(404, 'Package (e.g. code, parser, converter) does not exist')
@api.response(200, 'Metainfo send')
def get(self, metainfo_package_name):
'''
Get a JSON representation of the NOMAD Metainfo in its old legacy JSON format.
You can get the metainfo for 'common', and parser/code metainfo packages.
Parser/code packages constain the necessary definitions that the respective
parser/code might use. 'Common' contains all non specific general definitions.
Other required packages might also be returned, e.g. a parser might organize its
definitions in multiple packages.
'''
try:
metainfo = LegacyMetainfoEnvironment.from_legacy_package_path(metainfo_package_name)
except (ImportError, KeyError, FileNotFoundError, AttributeError):
abort(404, message='Metainfo package %s does not exist.' % metainfo_package_name)
if isinstance(metainfo, LegacyMetainfoEnvironment):
return metainfo.to_legacy_dict(metainfo.packages)
else:
abort(404, message='Metainfo package %s is not a legacy package.' % metainfo_package_name)
......@@ -267,7 +267,7 @@ class UploadListResource(Resource):
user = g.user
from_oasis = oasis_upload_id is not None
if from_oasis:
if not g.user.is_oasis_admin:
if not g.user.full_user().is_oasis_admin:
abort(401, 'Only an oasis admin can perform an oasis upload.')
if oasis_uploader_id is None:
abort(400, 'You must provide the original uploader for an oasis upload.')
......@@ -281,7 +281,7 @@ class UploadListResource(Resource):
uploader_id = request.args.get('uploader_id')
if uploader_id is not None:
if not g.user.is_admin:
if not g.user.full_user().is_admin:
abort(401, 'Only an admins can upload for other users.')
user = datamodel.User.get(user_id=uploader_id)
......@@ -615,10 +615,10 @@ class UploadCommandResource(Resource):
upload_command_form = 'curl "%s" -X PUT -F file=@<local_file>' % upload_url
upload_command_with_name = 'curl "%s" -X PUT -T <local_file>' % upload_url_with_name
upload_command_with_name = 'curl "%s" -T <local_file>' % upload_url_with_name
upload_progress_command = upload_command + ' | xargs echo'
upload_tar_command = 'tar -cf - <local_folder> | curl -# -H "%s" -T - | xargs echo' % upload_url
upload_tar_command = 'tar -cf - <local_folder> | curl "%s" -T - | xargs echo' % upload_url
return dict(
upload_url=upload_url,
......
......@@ -67,5 +67,5 @@ class Catalog(Resource):
es_response = es_search.execute()
mapping = Mapping()
mapping.map_catalog(es_response.hits, after, modified_since)
mapping.map_catalog(es_response.hits, after, modified_since, slim=False)
return rdf_respose(mapping.g)
......@@ -40,16 +40,16 @@ def get_optional_entry_prop(entry, name):
class Mapping():
def __init__(self):
self.g = Graph()
self.g.namespace_manager.bind('rdf', RDF)
self.g.namespace_manager.bind('dcat', DCAT)
self.g.namespace_manager.bind('dct', DCT)
self.g.namespace_manager.bind('vcard', VCARD)
self.g.namespace_manager.bind('foaf', FOAF)
self.g.namespace_manager.bind('hydra', HYDRA)
self.g.bind('rdf', RDF)
self.g.bind('dcat', DCAT)
self.g.bind('dct', DCT)
self.g.bind('vcard', VCARD)
self.g.bind('foaf', FOAF)
self.g.bind('hydra', HYDRA)
self.persons = {}
def map_catalog(self, entries, after: str, modified_since):
def map_catalog(self, entries, after: str, modified_since, slim=True):
def uri_ref(after):
kwargs = dict()
if after is not None:
......@@ -64,7 +64,7 @@ class Mapping():
self.g.add((catalog, RDF.type, DCAT.Catalog))
last_entry = None
for entry in entries:
self.g.add((catalog, DCT.dataset, self.map_entry(entry, slim=True)))
self.g.add((catalog, DCT.dataset, self.map_entry(entry, slim=slim)))
last_entry = entry
hydra_collection = uri_ref(after)
......@@ -76,6 +76,9 @@ class Mapping():
self.g.add((hydra_collection, RDF.type, HYDRA.collection))
for person in self.persons.values():
self.g.add((catalog, DCT.creator, person))
def map_entry(self, entry: EntryMetadata, slim=False):
dataset = URIRef(url('datasets', entry.calc_id))
......
......@@ -13,6 +13,7 @@ sys.modules['optimade.server.logger'] = importlib.import_module('nomad.app.optim
from nomad import config, utils # nopep8
from optimade.server.config import CONFIG # nopep8
CONFIG.root_path = '%s/optimade' % config.services.api_base_path
CONFIG.base_url = config.api_url(api='optimade')
from optimade.server import main as optimade # nopep8
from optimade.server.routers import structures # nopep8
......
......@@ -23,7 +23,7 @@ import traceback
from nomad import config, utils
from .common import root_path
from .routers import users, entries, auth, datasets
from .routers import auth, users, entries, datasets, uploads
logger = utils.get_logger(__name__)
......@@ -152,3 +152,4 @@ app.include_router(auth.router, prefix='/auth')
app.include_router(users.router, prefix='/users')
app.include_router(entries.router, prefix='/entries')
app.include_router(datasets.router, prefix='/datasets')
app.include_router(uploads.router, prefix='/uploads')
......@@ -515,6 +515,12 @@ class Pagination(BaseModel):
@root_validator(skip_on_failure=True)
def validate_values(cls, values): # pylint: disable=no-self-argument
# Because of a bug in pydantic (#2670), root validators can't be overridden, so
# we invoke a class method, which *can* be overridden.
return cls._root_validation(values)
@classmethod
def _root_validation(cls, values):
page = values.get('page')
page_after_value = values.get('page_after_value')
page_size = values.get('page_size')
......@@ -579,8 +585,8 @@ class PaginationResponse(Pagination):
# No validation - behaviour of this field depends on api method
return page_after_value
@root_validator(skip_on_failure=True)
def validate_values(cls, values): # pylint: disable=no-self-argument
@classmethod
def _root_validation(cls, values): # pylint: disable=no-self-argument
# No validation
return values
......@@ -620,6 +626,11 @@ class PaginationResponse(Pagination):
self.next_page_after_value = None
else:
self.next_page_after_value = str(ind + self.page_size - 1)
if self.page < 1 or (
self.total == 0 and self.page != 1) or (
self.total > 0 and (self.page - 1) * self.page_size >= self.total):
raise HTTPException(400, detail='Page out of range requested.')
if request.method.upper() == 'GET':
self.populate_urls(request)
......
......@@ -16,12 +16,17 @@
# limitations under the License.
#
from typing import cast
from fastapi import Depends, APIRouter, HTTPException, status
import hmac
import hashlib
import uuid
from typing import Callable, cast
from inspect import Parameter, signature
from functools import wraps
from fastapi import APIRouter, Depends, Query as FastApiQuery, HTTPException, status
from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
from pydantic import BaseModel
from nomad import infrastructure, config, datamodel
from nomad import utils, infrastructure, config, datamodel
from nomad.utils import get_logger, strip
from ..common import root_path
......@@ -42,48 +47,142 @@ class Token(BaseModel):
oauth2_scheme = OAuth2PasswordBearer(tokenUrl=f'{root_path}/auth/token', auto_error=False)
async def get_optional_user(access_token: str = Depends(oauth2_scheme)) -> User:
def create_user_dependency(
required: bool = False,
basic_auth_allowed: bool = False,
bearer_token_auth_allowed: bool = True,
upload_token_auth_allowed: bool = False) -> Callable:
'''
A dependency that provides the authenticated (if credentials are available) or None.
Creates a dependency for getting the authenticated user. The parameters define if
the authentication is required or not, and which authentication methods are allowed.
'''
if access_token is None:
user: datamodel.User = None
else:
try:
user = cast(datamodel.User, infrastructure.keycloak.tokenauth(access_token))
except infrastructure.KeycloakError as e:
def user_dependency(**kwargs) -> User:
user = None
if basic_auth_allowed:
user = _get_user_basic_auth(kwargs.get('form_data'))
if not user and bearer_token_auth_allowed:
user = _get_user_bearer_token_auth(kwargs.get('bearer_token'))
if not user and upload_token_auth_allowed:
user = _get_user_upload_token_auth(kwargs.get('token'))
if required and not user:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=str(e), headers={'WWW-Authenticate': 'Bearer'})
detail='Authorization required.')
if config.oasis.allowed_users is not None:
# We're an oasis, and have allowed_users set
if not user:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail='Authentication is required for this Oasis',
headers={'WWW-Authenticate': 'Bearer'})
if user.email not in config.oasis.allowed_users:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail='You are not authorized to access this Oasis',
headers={'WWW-Authenticate': 'Bearer'})
return user
# Create the desired function signature (as it depends on which auth options are allowed)
new_parameters = []
if basic_auth_allowed:
new_parameters.append(
Parameter(
name='form_data',
annotation=OAuth2PasswordRequestForm,
default=Depends(),
kind=Parameter.KEYWORD_ONLY))
if bearer_token_auth_allowed:
new_parameters.append(
Parameter(
name='bearer_token',
annotation=str,
default=Depends(oauth2_scheme),
kind=Parameter.KEYWORD_ONLY))
if upload_token_auth_allowed:
new_parameters.append(
Parameter(
name='token',
annotation=str,
default=FastApiQuery(
None,
description='Token for simplified authorization for uploading.'),
kind=Parameter.KEYWORD_ONLY))
# Create a wrapper around user_dependency, and set the signature on it
@wraps(user_dependency)
def wrapper(**kwargs) -> Callable:
return user_dependency(**kwargs)
if config.oasis.allowed_users is not None:
if user is None:
sig = signature(user_dependency)
sig = sig.replace(parameters=tuple(new_parameters))
wrapper.__signature__ = sig # type: ignore
return wrapper
def _get_user_basic_auth(form_data: OAuth2PasswordRequestForm) -> User:
'''
Verifies basic auth (username and password), throwing an exception if illegal credentials
are provided, and returns the corresponding user object if successful, None if no
credentials provided.
'''
if form_data and form_data.username and form_data.password:
try:
infrastructure.keycloak.basicauth(form_data.username, form_data.password)
user = cast(datamodel.User, infrastructure.keycloak.get_user(form_data.username))
return user
except infrastructure.KeycloakError:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail='Authentication is required for this Oasis',
detail='Incorrect username or password',
headers={'WWW-Authenticate': 'Bearer'})
return None
if user.email not in config.oasis.allowed_users:
def _get_user_bearer_token_auth(bearer_token: str) -> User:
'''
Verifies bearer_token (throwing exception if illegal value provided) and returns the
corresponding user object, or None, if no bearer_token provided.
'''
if bearer_token:
try:
user = cast(datamodel.User, infrastructure.keycloak.tokenauth(bearer_token))
return user
except infrastructure.KeycloakError as e:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail='You are not authorized to access this Oasis',
headers={'WWW-Authenticate': 'Bearer'})
return user
detail=str(e), headers={'WWW-Authenticate': 'Bearer'})
return None
async def get_required_user(user: User = Depends(get_optional_user)) -> User:
def _get_user_upload_token_auth(upload_token: str) -> User:
'''
A dependency that provides the authenticated user or raises 401 if no user is
authenticated.
Verifies the upload token (throwing exception if illegal value provided) and returns the
corresponding user object, or None, if no upload_token provided.
'''
if user is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail='Authentication required',
headers={'WWW-Authenticate': 'Bearer'})
if upload_token:
try:
payload, signature = upload_token.split('.')
payload_bytes = utils.base64_decode(payload)
signature_bytes = utils.base64_decode(signature)
compare = hmac.new(
bytes(config.services.api_secret, 'utf-8'),
msg=payload_bytes,
digestmod=hashlib.sha1)
return user
if signature_bytes == compare.digest():
user_id = str(uuid.UUID(bytes=payload_bytes))
user = cast(datamodel.User, infrastructure.keycloak.get_user(user_id))
return user
except Exception:
# Decode error, format error, user not found, etc.
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail='A invalid upload token was supplied.')
return None
_bad_credentials_response = status.HTTP_401_UNAUTHORIZED, {
......@@ -146,3 +245,15 @@ async def get_token_via_query(username: str, password: str):
headers={'WWW-Authenticate': 'Bearer'})
return {'access_token': access_token, 'token_type': 'bearer'}
def generate_upload_token(user):
payload = uuid.UUID(user.user_id).bytes
signature = hmac.new(
bytes(config.services.api_secret, 'utf-8'),
msg=payload,
digestmod=hashlib.sha1)
return '%s.%s' % (
utils.base64_encode(payload),
utils.base64_encode(signature.digest()))
......@@ -30,7 +30,7 @@ from nomad.utils import strip, create_uuid
from nomad.datamodel import Dataset as DatasetDefinitionCls
from nomad.doi import DOI
from .auth import get_required_user
from .auth import create_user_dependency
from .entries import _do_exaustive_search
from ..utils import create_responses, parameter_dependency_from_model
from ..models import (
......@@ -72,7 +72,7 @@ _existing_name_response = status.HTTP_400_BAD_REQUEST, {
_dataset_is_fixed_response = status.HTTP_400_BAD_REQUEST, {
'model': HTTPExceptionModel,
'description': strip('''
The dataset already as a DOI and cannot be changed anymore.
The dataset already has a DOI and cannot be changed anymore.
''')}
......@@ -191,7 +191,7 @@ async def get_dataset(
response_model_exclude_unset=True,
response_model_exclude_none=True)
async def post_datasets(
create: DatasetCreate, user: User = Depends(get_required_user)):
create: DatasetCreate, user: User = Depends(create_user_dependency(required=True))):
'''
Create a new dataset.
'''
......@@ -262,7 +262,7 @@ async def post_datasets(
response_model_exclude_none=True)
async def delete_dataset(
dataset_id: str = Path(..., description='The unique dataset id of the dataset to delete.'),
user: User = Depends(get_required_user)):
user: User = Depends(create_user_dependency(required=True))):
'''
Delete an dataset.
'''
......@@ -327,7 +327,7 @@ async def delete_dataset(
response_model_exclude_none=True)
async def assign_doi(
dataset_id: str = Path(..., description='The unique dataset id of the dataset to delete.'),
user: User = Depends(get_required_user)):
user: User = Depends(create_user_dependency(required=True))):
'''
Assign a DOI to a dataset.
'''
......
......@@ -31,7 +31,7 @@ from nomad import search, files, config, utils
from nomad.utils import strip
from nomad.archive import RequiredReader, RequiredValidationError, ArchiveQueryError
from .auth import get_optional_user
from .auth import create_user_dependency
from ..utils import create_streamed_zipfile, File, create_responses
from ..models import (
EntryPagination, WithQuery, MetadataRequired, EntriesMetadataResponse, EntriesMetadata,
......@@ -116,7 +116,7 @@ def perform_search(*args, **kwargs):
async def post_entries_metadata_query(
request: Request,
data: EntriesMetadata,
user: User = Depends(get_optional_user)):
user: User = Depends(create_user_dependency())):
'''
Executes a *query* and returns a *page* of the results with *required* result data
......@@ -157,7 +157,7 @@ async def get_entries_metadata(
with_query: WithQuery = Depends(query_parameters),
pagination: EntryPagination = Depends(entry_pagination_parameters),
required: MetadataRequired = Depends(metadata_required_parameters),
user: User = Depends(get_optional_user)):
user: User = Depends(create_user_dependency())):
'''
Executes a *query* and returns a *page* of the results with *required* result data.
This is a version of `/entries/query`. Queries work a little different, because
......@@ -367,7 +367,7 @@ _entries_raw_query_docstring = strip('''
response_model_exclude_unset=True,
response_model_exclude_none=True)
async def post_entries_raw_query(
request: Request, data: EntriesRaw, user: User = Depends(get_optional_user)):
request: Request, data: EntriesRaw, user: User = Depends(create_user_dependency())):
return _answer_entries_raw_request(
owner=data.owner, query=data.query, pagination=data.pagination, user=user)
......@@ -386,7 +386,7 @@ async def get_entries_raw(
request: Request,
with_query: WithQuery = Depends(query_parameters),
pagination: EntryPagination = Depends(entry_pagination_parameters),
user: User = Depends(get_optional_user)):
user: User = Depends(create_user_dependency())):
res = _answer_entries_raw_request(
owner=with_query.owner, query=with_query.query, pagination=pagination, user=user)
......@@ -422,7 +422,7 @@ _entries_raw_download_query_docstring = strip('''
response_class=StreamingResponse,
responses=create_responses(_raw_download_response, _bad_owner_response))
async def post_entries_raw_download_query(
data: EntriesRawDownload, user: User = Depends(get_optional_user)):
data: EntriesRawDownload, user: User = Depends(create_user_dependency())):
return _answer_entries_raw_download_request(
owner=data.owner, query=data.query, files=data.files, user=user)
......@@ -438,7 +438,7 @@ async def post_entries_raw_download_query(
async def get_entries_raw_download(
with_query: WithQuery = Depends(query_parameters),
files: Files = Depends(files_parameters),
user: User = Depends(get_optional_user)):
user: User = Depends(create_user_dependency())):
return _answer_entries_raw_download_request(
owner=with_query.owner, query=with_query.query, files=files, user=user)
......@@ -536,7 +536,7 @@ _entries_archive_docstring = strip('''
response_model_exclude_none=True,
responses=create_responses(_bad_owner_response, _bad_archive_required_response))
async def post_entries_archive_query(
request: Request, data: EntriesArchive, user: User = Depends(get_optional_user)):
request: Request, data: EntriesArchive, user: User = Depends(create_user_dependency())):
return _answer_entries_archive_request(
owner=data.owner, query=data.query, pagination=data.pagination,
......@@ -556,7 +556,7 @@ async def get_entries_archive_query(
request: Request,
with_query: WithQuery = Depends(query_parameters),
pagination: EntryPagination = Depends(entry_pagination_parameters),
user: User = Depends(get_optional_user)):
user: User = Depends(create_user_dependency())):
res = _answer_entries_archive_request(
owner=with_query.owner, query=with_query.query, pagination=pagination,
......@@ -641,7 +641,7 @@ _entries_archive_download_docstring = strip('''
responses=create_responses(
_archive_download_response, _bad_owner_response, _bad_archive_required_response))
async def post_entries_archive_download_query(
data: EntriesArchiveDownload, user: User = Depends(get_optional_user)):
data: EntriesArchiveDownload, user: User = Depends(create_user_dependency())):
return _answer_entries_archive_download_request(
owner=data.owner, query=data.query, files=data.files, user=user)
......@@ -658,7 +658,7 @@ async def post_entries_archive_download_query(
async def get_entries_archive_download(
with_query: WithQuery = Depends(query_parameters),
files: Files = Depends(files_parameters),
user: User = Depends(get_optional_user)):
user: User = Depends(create_user_dependency())):
return _answer_entries_archive_download_request(
owner=with_query.owner, query=with_query.query, files=files, user=user)
......@@ -674,7 +674,7 @@ async def get_entries_archive_download(
async def get_entry_metadata(
entry_id: str = Path(..., description='The unique entry id of the entry to retrieve metadata from.'),