Commit 44ec6189 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Moved new v1 api into a fastapi sub application.

parent 32846c39
......@@ -16,110 +16,22 @@
# limitations under the License.
#
from fastapi import FastAPI, status, Request
from fastapi.responses import JSONResponse, RedirectResponse
from fastapi import FastAPI
from fastapi.middleware.wsgi import WSGIMiddleware
import traceback
from nomad import config, utils
from nomad import config
from .flask import app as flask_app
from .routers import users, entries, auth, datasets
from .optimade import optimade_app
from .flask import app as flask_app
from .v1.main import app as v1_app
logger = utils.get_logger(__name__)
app = FastAPI(
openapi_url='%s/api/v1/openapi.json' % config.services.api_base_path,
docs_url='%s/api/v1/extensions/docs' % config.services.api_base_path,
redoc_url='%s/api/v1/extensions/redoc' % config.services.api_base_path,
swagger_ui_oauth2_redirect_url='%s/api/v1/docs/oauth2-redirect' % config.services.api_base_path,
title='NOMAD API',
version='v1, NOMAD %s@%s' % (config.meta.version, config.meta.commit),
description=utils.strip('''
**Disclaimer!** This is the new NOMAD API. It is still under development and only includes a
part of the NOMAD API functionality. You can still use the old flask-based API
as `/api` and the optimade API as `/optimade/v1`.
## Getting started
... TODO put the examples and tutorial here ...
## Conventions
### Paths
The various API operations are organized with the following path scheme. The first
part of the path, describes the data entity that is covered by
the operations below (e.g. `entries`, `users`, `datasets`, `uploads`). For example
everything below `entries` will be about searching entries, getting
an entry, editing entries, etc.
The second (optional and variable) path segment allows to denote a specific entity instance,
e.g. a specific entry or dataset, usually by id. With out such a variable second
path segment, its about all instances, e.g. searching entries or listing all datasets.
Optional (if available) further path segments will determine the variety and format
of data. This is mostly for entries to distinguish the metadata, raw, and archive
data or distinguish between listing (i.e. paginated json) and downloading
(i.e. streaming a zip-file)
Further, we try to adhere to the paradim of getting and posting resources. Therefore,
when you post a complex query, you will not post it to `/entries` (a query is not an entry),
but `/entries/query`. Here *query* being a kind of virtual resource.
### Parameters and bodies for GET and POST operations
We offer **GET** and **POST** versions for many complex operations. The idea is that
**GET** is easy to use, e.g. via curl or simply in the browser, while **POST**
allows to provide more complex parameters (i.e. a JSON body). For example to
search for entries, you can use the **GET** operation `/entries` to specify simple
queries via URL, e.g. `/entries?code_name=VASP&atoms=Ti`, but you would use
**POST** `/entries/query` to provide a complex nested queries, e.g. with logical
operators.
Typicall the **POST** version is a super-set of the functionality of the **GET**
version. But, most top-level parameters in the **POST** body, will be available
in the **GET** version as URL parameters with the same name and meaning. This
is especially true for reoccuring parameters for general API concepts like pagination
or specifying required result fields.
### Response layout
Typically a response will mirror all input parameters in the normalized form that
was used to perform the operation.
Some of these will be augmented with result values. For example the pagination
section of a request will be augmented with the total available number.
The actual requested data, will be placed under the key `data`.
## About Authentication
NOMAD is an open datasharing platform, and most of the API operations do not require
any authorization and can be freely used without a user or credentials. However,
to upload data, edit data, or view your own and potentially unpublished data,
the API needs to authenticate you.
The NOMAD API uses OAuth and tokens to authenticate users. We provide simple operations
that allow you to acquire an *access token* via username and password based
authentication (`/auth/token`). The resulting access token can then be used on all operations
(e.g. that support or require authentication).
To use authentication in the dashboard, simply use the Authorize button. The
dashboard GUI will manage the access token and use it while you try out the various
operations.
'''))
async def redirect_to_docs(req: Request):
return RedirectResponse('%s/api/v1/extensions/docs' % config.services.api_base_path)
app = FastAPI()
app.add_route('%s/api/v1' % config.services.api_base_path, redirect_to_docs, include_in_schema=False)
app.add_route('%s/api/v1/' % config.services.api_base_path, redirect_to_docs, include_in_schema=False)
app_base = config.services.api_base_path
app.mount(f'{app_base}/api/v1', v1_app)
app.mount(f'{app_base}/optimade', optimade_app)
app.mount(app_base, WSGIMiddleware(flask_app))
@app.on_event('startup')
......@@ -133,27 +45,3 @@ async def startup_event():
pass
infrastructure.setup()
@app.exception_handler(Exception)
async def unicorn_exception_handler(request: Request, e: Exception):
logger.error('unexpected exception in API', url=request.url, exc_info=e)
return JSONResponse(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
content={
'detail': {
'reason': 'Unexpected exception while handling your request',
'exception': str(e),
'exception_class': e.__class__.__name__,
'exception_traceback': traceback.format_exc()
}
}
)
app.include_router(auth.router, prefix='%s/api/v1/auth' % config.services.api_base_path)
app.include_router(users.router, prefix='%s/api/v1/users' % config.services.api_base_path)
app.include_router(entries.router, prefix='%s/api/v1/entries' % config.services.api_base_path)
app.include_router(datasets.router, prefix='%s/api/v1/datasets' % config.services.api_base_path)
app.mount('%s/optimade' % config.services.api_base_path, optimade_app)
app.mount(config.services.api_base_path, WSGIMiddleware(flask_app))
......@@ -16,46 +16,6 @@
# limitations under the License.
#
from typing import Iterator
import os.path
import zipfile
from nomad import config
from nomad.datamodel import EntryArchive, EntryMetadata
from nomad.app.utils import create_streamed_zipfile, File
from tests.conftest import clear_raw_files
from tests.test_files import create_test_upload_files
def test_create_streamed_zip(raw_files_infra):
# We use the files of a simpe test upload to create streamed zip with all the raw
# files.
archive = EntryArchive()
metadata = archive.m_create(EntryMetadata)
metadata.upload_id = 'test_id'
metadata.calc_id = 'test_id'
metadata.mainfile = 'root/subdir/mainfile.json'
upload_files = create_test_upload_files('test_id', [archive])
def generate_files() -> Iterator[File]:
for path in upload_files.raw_file_manifest():
with upload_files.raw_file(path) as f:
yield File(
path=path,
f=f,
size=upload_files.raw_file_size(path))
if not os.path.exists(config.fs.tmp):
os.makedirs(config.fs.tmp)
zip_file_path = os.path.join(config.fs.tmp, 'results.zip')
with open(zip_file_path, 'wb') as f:
for content in create_streamed_zipfile(generate_files()):
f.write(content)
with zipfile.ZipFile(zip_file_path) as zf:
assert zf.testzip() is None
clear_raw_files()
root_path = f'{config.services.api_base_path}/api/v1'
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD. See https://nomad-lab.eu for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from fastapi import FastAPI, status, Request
from fastapi.responses import JSONResponse, RedirectResponse
import traceback
from nomad import config, utils
from .common import root_path
from .routers import users, entries, auth, datasets
logger = utils.get_logger(__name__)
app = FastAPI(
root_path=root_path,
openapi_url='/openapi.json',
docs_url='/extensions/docs',
redoc_url='/extensions/redoc',
swagger_ui_oauth2_redirect_url='/extensions/docs/oauth2-redirect',
title='NOMAD API',
version='v1, NOMAD %s@%s' % (config.meta.version, config.meta.commit),
description=utils.strip('''
**Disclaimer!** This is the new NOMAD API. It is still under development and only includes a
part of the NOMAD API functionality. You can still use the old flask-based API
as `/api` and the optimade API as `/optimade/v1`.
## Getting started
... TODO put the examples and tutorial here ...
## Conventions
### Paths
The various API operations are organized with the following path scheme. The first
part of the path, describes the data entity that is covered by
the operations below (e.g. `entries`, `users`, `datasets`, `uploads`). For example
everything below `entries` will be about searching entries, getting
an entry, editing entries, etc.
The second (optional and variable) path segment allows to denote a specific entity instance,
e.g. a specific entry or dataset, usually by id. With out such a variable second
path segment, its about all instances, e.g. searching entries or listing all datasets.
Optional (if available) further path segments will determine the variety and format
of data. This is mostly for entries to distinguish the metadata, raw, and archive
data or distinguish between listing (i.e. paginated json) and downloading
(i.e. streaming a zip-file)
Further, we try to adhere to the paradim of getting and posting resources. Therefore,
when you post a complex query, you will not post it to `/entries` (a query is not an entry),
but `/entries/query`. Here *query* being a kind of virtual resource.
### Parameters and bodies for GET and POST operations
We offer **GET** and **POST** versions for many complex operations. The idea is that
**GET** is easy to use, e.g. via curl or simply in the browser, while **POST**
allows to provide more complex parameters (i.e. a JSON body). For example to
search for entries, you can use the **GET** operation `/entries` to specify simple
queries via URL, e.g. `/entries?code_name=VASP&atoms=Ti`, but you would use
**POST** `/entries/query` to provide a complex nested queries, e.g. with logical
operators.
Typicall the **POST** version is a super-set of the functionality of the **GET**
version. But, most top-level parameters in the **POST** body, will be available
in the **GET** version as URL parameters with the same name and meaning. This
is especially true for reoccuring parameters for general API concepts like pagination
or specifying required result fields.
### Response layout
Typically a response will mirror all input parameters in the normalized form that
was used to perform the operation.
Some of these will be augmented with result values. For example the pagination
section of a request will be augmented with the total available number.
The actual requested data, will be placed under the key `data`.
## About Authentication
NOMAD is an open datasharing platform, and most of the API operations do not require
any authorization and can be freely used without a user or credentials. However,
to upload data, edit data, or view your own and potentially unpublished data,
the API needs to authenticate you.
The NOMAD API uses OAuth and tokens to authenticate users. We provide simple operations
that allow you to acquire an *access token* via username and password based
authentication (`/auth/token`). The resulting access token can then be used on all operations
(e.g. that support or require authentication).
To use authentication in the dashboard, simply use the Authorize button. The
dashboard GUI will manage the access token and use it while you try out the various
operations.
'''))
async def redirect_to_docs(req: Request):
return RedirectResponse(f'{root_path}/extensions/docs')
# app.add_route(f'{root_path}', redirect_to_docs, include_in_schema=False)
app.add_route('/', redirect_to_docs, include_in_schema=False)
@app.on_event('startup')
async def startup_event():
from nomad import infrastructure
# each subprocess is supposed disconnect connect again: https://jira.mongodb.org/browse/PYTHON-2090
try:
from mongoengine import disconnect
disconnect()
except Exception:
pass
infrastructure.setup()
@app.exception_handler(Exception)
async def unicorn_exception_handler(request: Request, e: Exception):
logger.error('unexpected exception in API', url=request.url, exc_info=e)
return JSONResponse(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
content={
'detail': {
'reason': 'Unexpected exception while handling your request',
'exception': str(e),
'exception_class': e.__class__.__name__,
'exception_traceback': traceback.format_exc()
}
}
)
app.include_router(auth.router, prefix='/auth')
app.include_router(users.router, prefix='/users')
app.include_router(entries.router, prefix='/entries')
app.include_router(datasets.router, prefix='/datasets')
......@@ -29,9 +29,10 @@ import fnmatch
from nomad import datamodel # pylint: disable=unused-import
from nomad.utils import strip
from nomad.metainfo import Datetime, MEnum
from nomad.app.utils import parameter_dependency_from_model
from nomad.metainfo.search_extension import metrics, search_quantities
from .utils import parameter_dependency_from_model
User = datamodel.User.m_def.a_pydantic.model
......
......@@ -22,8 +22,10 @@ from pydantic import BaseModel
from nomad import infrastructure
from nomad.utils import get_logger, strip
from nomad.app.models import User, HTTPExceptionModel
from nomad.app.utils import create_responses
from ..common import root_path
from ..models import User, HTTPExceptionModel
from ..utils import create_responses
logger = get_logger(__name__)
......@@ -36,7 +38,7 @@ class Token(BaseModel):
token_type: str
oauth2_scheme = OAuth2PasswordBearer(tokenUrl='/api/v1/auth/token', auto_error=False)
oauth2_scheme = OAuth2PasswordBearer(tokenUrl=f'{root_path}/auth/token', auto_error=False)
async def get_optional_user(access_token: str = Depends(oauth2_scheme)) -> User:
......
......@@ -27,13 +27,12 @@ from nomad.utils import strip, create_uuid
from nomad.datamodel import Dataset as DatasetDefinitionCls
from nomad.doi import DOI
from nomad.app.routers.auth import get_required_user
from nomad.app.utils import create_responses
from nomad.app.models import (
pagination_parameters, Pagination, PaginationResponse, Query,
HTTPExceptionModel, User, Direction, Owner)
from .auth import get_required_user
from .entries import _do_exaustive_search
from ..utils import create_responses
from ..models import (
pagination_parameters, Pagination, PaginationResponse, Query, HTTPExceptionModel,
User, Direction, Owner)
router = APIRouter()
......
......@@ -29,9 +29,10 @@ from nomad.utils import strip
from nomad.archive import (
query_archive, ArchiveQueryError, compute_required_with_referenced,
read_partial_archives_from_mongo, filter_archive)
from nomad.app.utils import create_streamed_zipfile, File, create_responses
from nomad.app.routers.auth import get_optional_user
from nomad.app.models import (
from .auth import get_optional_user
from ..utils import create_streamed_zipfile, File, create_responses
from ..models import (
Pagination, WithQuery, MetadataRequired, EntriesMetadataResponse, EntriesMetadata,
EntryMetadataResponse, query_parameters, metadata_required_parameters, Files, Query,
pagination_parameters, files_parameters, User, Owner, HTTPExceptionModel, EntriesRaw,
......
......@@ -18,11 +18,12 @@
from fastapi import Depends, APIRouter, status
from nomad.app.routers.auth import get_required_user
from nomad.app.models import User, HTTPExceptionModel
from nomad.app.utils import create_responses
from nomad.utils import strip
from .auth import get_required_user
from ..models import User, HTTPExceptionModel
from ..utils import create_responses
router = APIRouter()
default_tag = 'users'
......
......@@ -31,8 +31,8 @@ from nomad.datamodel.material import Material
from nomad import config, datamodel, infrastructure, utils
from nomad.metainfo.search_extension import ( # pylint: disable=unused-import
search_quantities, metrics, order_default_quantities, groups)
from nomad.app import models as api_models
from nomad.app.models import (
from nomad.app.v1 import models as api_models
from nomad.app.v1.models import (
Pagination, PaginationResponse, Query, MetadataRequired, SearchResponse, Aggregation,
Statistic, StatisticResponse, AggregationOrderType, AggregationResponse, AggregationDataItem)
......
......@@ -50,7 +50,7 @@ def admin_user_auth(admin_user: User):
@pytest.fixture(scope='session')
def client():
return TestClient(app, base_url='http://testserver/api/v1/')
return TestClient(app, base_url='http://testserver/')
@pytest.fixture(scope='module')
......
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD. See https://nomad-lab.eu for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from fastapi.testclient import TestClient
from nomad.app.main import app
@pytest.fixture(scope='session')
def client():
print('###')
return TestClient(app, base_url='http://testserver/api/v1/')
......@@ -23,7 +23,7 @@ import io
import json
from nomad.metainfo.search_extension import search_quantities
from nomad.app.models import AggregateableQuantity, Metric
from nomad.app.v1.models import AggregateableQuantity, Metric
from tests.utils import assert_at_least
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment