Moved new v1 api into a fastapi sub application.

nomad/app/main.py

@@ -16,110 +16,22 @@
 # limitations under the License.
 #
 
-from fastapi import FastAPI, status, Request
-from fastapi.responses import JSONResponse, RedirectResponse
+from fastapi import FastAPI
 from fastapi.middleware.wsgi import WSGIMiddleware
-import traceback
 
-from nomad import config, utils
+from nomad import config
 
-from .flask import app as flask_app
-from .routers import users, entries, auth, datasets
 from .optimade import optimade_app
+from .flask import app as flask_app
+from .v1.main import app as v1_app
 
 
-logger = utils.get_logger(__name__)
-
-app = FastAPI(
-    openapi_url='%s/api/v1/openapi.json' % config.services.api_base_path,
-    docs_url='%s/api/v1/extensions/docs' % config.services.api_base_path,
-    redoc_url='%s/api/v1/extensions/redoc' % config.services.api_base_path,
-    swagger_ui_oauth2_redirect_url='%s/api/v1/docs/oauth2-redirect' % config.services.api_base_path,
-
-    title='NOMAD API',
-    version='v1, NOMAD %s@%s' % (config.meta.version, config.meta.commit),
-    description=utils.strip('''
-        **Disclaimer!** This is the new NOMAD API. It is still under development and only includes a
-        part of the NOMAD API functionality. You can still use the old flask-based API
-        as `/api` and the optimade API as `/optimade/v1`.
-
-        ## Getting started
-
-        ... TODO put the examples and tutorial here ...
-
-        ## Conventions
-
-        ### Paths
-
-        The various API operations are organized with the following path scheme. The first
-        part of the path, describes the data entity that is covered by
-        the operations below (e.g. `entries`, `users`, `datasets`, `uploads`). For example
-        everything below `entries` will be about searching entries, getting
-        an entry, editing entries, etc.
-
-        The second (optional and variable) path segment allows to denote a specific entity instance,
-        e.g. a specific entry or dataset, usually by id. With out such a variable second
-        path segment, its about all instances, e.g. searching entries or listing all datasets.
-
-        Optional (if available) further path segments will determine the variety and format
-        of data. This is mostly for entries to distinguish the metadata, raw, and archive
-        data or distinguish between listing (i.e. paginated json) and downloading
-        (i.e. streaming a zip-file)
-
-        Further, we try to adhere to the paradim of getting and posting resources. Therefore,
-        when you post a complex query, you will not post it to `/entries` (a query is not an entry),
-        but `/entries/query`. Here *query* being a kind of virtual resource.
-
-        ### Parameters and bodies for GET and POST operations
-
-        We offer **GET** and **POST** versions for many complex operations. The idea is that
-        **GET** is easy to use, e.g. via curl or simply in the browser, while **POST**
-        allows to provide more complex parameters (i.e. a JSON body). For example to
-        search for entries, you can use the **GET** operation `/entries` to specify simple
-        queries via URL, e.g. `/entries?code_name=VASP&atoms=Ti`, but you would use
-        **POST** `/entries/query` to provide a complex nested queries, e.g. with logical
-        operators.
-
-        Typicall the **POST** version is a super-set of the functionality of the **GET**
-        version. But, most top-level parameters in the **POST** body, will be available
-        in the **GET** version as URL parameters with the same name and meaning. This
-        is especially true for reoccuring parameters for general API concepts like pagination
-        or specifying required result fields.
-
-        ### Response layout
-
-        Typically a response will mirror all input parameters in the normalized form that
-        was used to perform the operation.
-
-        Some of these will be augmented with result values. For example the pagination
-        section of a request will be augmented with the total available number.
-
-        The actual requested data, will be placed under the key `data`.
-
-        ## About Authentication
-
-        NOMAD is an open datasharing platform, and most of the API operations do not require
-        any authorization and can be freely used without a user or credentials. However,
-        to upload data, edit data, or view your own and potentially unpublished data,
-        the API needs to authenticate you.
-
-        The NOMAD API uses OAuth and tokens to authenticate users. We provide simple operations
-        that allow you to acquire an *access token* via username and password based
-        authentication (`/auth/token`). The resulting access token can then be used on all operations
-        (e.g. that support or require authentication).
-
-        To use authentication in the dashboard, simply use the Authorize button. The
-        dashboard GUI will manage the access token and use it while you try out the various
-        operations.
-    '''))
-
-
-async def redirect_to_docs(req: Request):
-    return RedirectResponse('%s/api/v1/extensions/docs' % config.services.api_base_path)
-
+app = FastAPI()
 
-app.add_route('%s/api/v1' % config.services.api_base_path, redirect_to_docs, include_in_schema=False)
-app.add_route('%s/api/v1/' % config.services.api_base_path, redirect_to_docs, include_in_schema=False)
+app_base = config.services.api_base_path
+app.mount(f'{app_base}/api/v1', v1_app)
+app.mount(f'{app_base}/optimade', optimade_app)
+app.mount(app_base, WSGIMiddleware(flask_app))
 
 
 @app.on_event('startup')
@@ -133,27 +45,3 @@ async def startup_event():
         pass
 
     infrastructure.setup()
-
-
-@app.exception_handler(Exception)
-async def unicorn_exception_handler(request: Request, e: Exception):
-    logger.error('unexpected exception in API', url=request.url, exc_info=e)
-    return JSONResponse(
-        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-        content={
-            'detail': {
-                'reason': 'Unexpected exception while handling your request',
-                'exception': str(e),
-                'exception_class': e.__class__.__name__,
-                'exception_traceback': traceback.format_exc()
-            }
-        }
-    )
-
-app.include_router(auth.router, prefix='%s/api/v1/auth' % config.services.api_base_path)
-app.include_router(users.router, prefix='%s/api/v1/users' % config.services.api_base_path)
-app.include_router(entries.router, prefix='%s/api/v1/entries' % config.services.api_base_path)
-app.include_router(datasets.router, prefix='%s/api/v1/datasets' % config.services.api_base_path)
-
-app.mount('%s/optimade' % config.services.api_base_path, optimade_app)
-app.mount(config.services.api_base_path, WSGIMiddleware(flask_app))

tests/app/test_utils.py → nomad/app/v1/common.py

@@ -16,46 +16,6 @@
 # limitations under the License.
 #
 
-from typing import Iterator
-import os.path
-import zipfile
-
 from nomad import config
-from nomad.datamodel import EntryArchive, EntryMetadata
-from nomad.app.utils import create_streamed_zipfile, File
-
-from tests.conftest import clear_raw_files
-from tests.test_files import create_test_upload_files
-
-
-def test_create_streamed_zip(raw_files_infra):
-    # We use the files of a simpe test upload to create streamed zip with all the raw
-    # files.
-    archive = EntryArchive()
-    metadata = archive.m_create(EntryMetadata)
-    metadata.upload_id = 'test_id'
-    metadata.calc_id = 'test_id'
-    metadata.mainfile = 'root/subdir/mainfile.json'
-
-    upload_files = create_test_upload_files('test_id', [archive])
-
-    def generate_files() -> Iterator[File]:
-        for path in upload_files.raw_file_manifest():
-            with upload_files.raw_file(path) as f:
-                yield File(
-                    path=path,
-                    f=f,
-                    size=upload_files.raw_file_size(path))
-
-    if not os.path.exists(config.fs.tmp):
-        os.makedirs(config.fs.tmp)
-
-    zip_file_path = os.path.join(config.fs.tmp, 'results.zip')
-    with open(zip_file_path, 'wb') as f:
-        for content in create_streamed_zipfile(generate_files()):
-            f.write(content)
-
-    with zipfile.ZipFile(zip_file_path) as zf:
-        assert zf.testzip() is None
 
-    clear_raw_files()
+root_path = f'{config.services.api_base_path}/api/v1'

nomad/app/v1/main.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from fastapi import FastAPI, status, Request
+from fastapi.responses import JSONResponse, RedirectResponse
+import traceback
+
+from nomad import config, utils
+
+from .common import root_path
+from .routers import users, entries, auth, datasets
+
+
+logger = utils.get_logger(__name__)
+
+app = FastAPI(
+    root_path=root_path,
+    openapi_url='/openapi.json',
+    docs_url='/extensions/docs',
+    redoc_url='/extensions/redoc',
+    swagger_ui_oauth2_redirect_url='/extensions/docs/oauth2-redirect',
+    title='NOMAD API',
+    version='v1, NOMAD %s@%s' % (config.meta.version, config.meta.commit),
+    description=utils.strip('''
+        **Disclaimer!** This is the new NOMAD API. It is still under development and only includes a
+        part of the NOMAD API functionality. You can still use the old flask-based API
+        as `/api` and the optimade API as `/optimade/v1`.
+
+        ## Getting started
+
+        ... TODO put the examples and tutorial here ...
+
+        ## Conventions
+
+        ### Paths
+
+        The various API operations are organized with the following path scheme. The first
+        part of the path, describes the data entity that is covered by
+        the operations below (e.g. `entries`, `users`, `datasets`, `uploads`). For example
+        everything below `entries` will be about searching entries, getting
+        an entry, editing entries, etc.
+
+        The second (optional and variable) path segment allows to denote a specific entity instance,
+        e.g. a specific entry or dataset, usually by id. With out such a variable second
+        path segment, its about all instances, e.g. searching entries or listing all datasets.
+
+        Optional (if available) further path segments will determine the variety and format
+        of data. This is mostly for entries to distinguish the metadata, raw, and archive
+        data or distinguish between listing (i.e. paginated json) and downloading
+        (i.e. streaming a zip-file)
+
+        Further, we try to adhere to the paradim of getting and posting resources. Therefore,
+        when you post a complex query, you will not post it to `/entries` (a query is not an entry),
+        but `/entries/query`. Here *query* being a kind of virtual resource.
+
+        ### Parameters and bodies for GET and POST operations
+
+        We offer **GET** and **POST** versions for many complex operations. The idea is that
+        **GET** is easy to use, e.g. via curl or simply in the browser, while **POST**
+        allows to provide more complex parameters (i.e. a JSON body). For example to
+        search for entries, you can use the **GET** operation `/entries` to specify simple
+        queries via URL, e.g. `/entries?code_name=VASP&atoms=Ti`, but you would use
+        **POST** `/entries/query` to provide a complex nested queries, e.g. with logical
+        operators.
+
+        Typicall the **POST** version is a super-set of the functionality of the **GET**
+        version. But, most top-level parameters in the **POST** body, will be available
+        in the **GET** version as URL parameters with the same name and meaning. This
+        is especially true for reoccuring parameters for general API concepts like pagination
+        or specifying required result fields.
+
+        ### Response layout
+
+        Typically a response will mirror all input parameters in the normalized form that
+        was used to perform the operation.
+
+        Some of these will be augmented with result values. For example the pagination
+        section of a request will be augmented with the total available number.
+
+        The actual requested data, will be placed under the key `data`.
+
+        ## About Authentication
+
+        NOMAD is an open datasharing platform, and most of the API operations do not require
+        any authorization and can be freely used without a user or credentials. However,
+        to upload data, edit data, or view your own and potentially unpublished data,
+        the API needs to authenticate you.
+
+        The NOMAD API uses OAuth and tokens to authenticate users. We provide simple operations
+        that allow you to acquire an *access token* via username and password based
+        authentication (`/auth/token`). The resulting access token can then be used on all operations
+        (e.g. that support or require authentication).
+
+        To use authentication in the dashboard, simply use the Authorize button. The
+        dashboard GUI will manage the access token and use it while you try out the various
+        operations.
+    '''))
+
+
+async def redirect_to_docs(req: Request):
+    return RedirectResponse(f'{root_path}/extensions/docs')
+
+
+# app.add_route(f'{root_path}', redirect_to_docs, include_in_schema=False)
+app.add_route('/', redirect_to_docs, include_in_schema=False)
+
+
+@app.on_event('startup')
+async def startup_event():
+    from nomad import infrastructure
+    # each subprocess is supposed disconnect connect again: https://jira.mongodb.org/browse/PYTHON-2090
+    try:
+        from mongoengine import disconnect
+        disconnect()
+    except Exception:
+        pass
+
+    infrastructure.setup()
+
+
+@app.exception_handler(Exception)
+async def unicorn_exception_handler(request: Request, e: Exception):
+    logger.error('unexpected exception in API', url=request.url, exc_info=e)
+    return JSONResponse(
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        content={
+            'detail': {
+                'reason': 'Unexpected exception while handling your request',
+                'exception': str(e),
+                'exception_class': e.__class__.__name__,
+                'exception_traceback': traceback.format_exc()
+            }
+        }
+    )
+
+app.include_router(auth.router, prefix='/auth')
+app.include_router(users.router, prefix='/users')
+app.include_router(entries.router, prefix='/entries')
+app.include_router(datasets.router, prefix='/datasets')

nomad/app/models.py → nomad/app/v1/models.py

@@ -29,9 +29,10 @@ import fnmatch
 from nomad import datamodel  # pylint: disable=unused-import
 from nomad.utils import strip
 from nomad.metainfo import Datetime, MEnum
-from nomad.app.utils import parameter_dependency_from_model
 from nomad.metainfo.search_extension import metrics, search_quantities
 
+from .utils import parameter_dependency_from_model
+
 
 User = datamodel.User.m_def.a_pydantic.model
 

nomad/app/routers/auth.py → nomad/app/v1/routers/auth.py

@@ -22,8 +22,10 @@ from pydantic import BaseModel
 
 from nomad import infrastructure
 from nomad.utils import get_logger, strip
-from nomad.app.models import User, HTTPExceptionModel
-from nomad.app.utils import create_responses
+
+from ..common import root_path
+from ..models import User, HTTPExceptionModel
+from ..utils import create_responses
 
 logger = get_logger(__name__)
 
@@ -36,7 +38,7 @@ class Token(BaseModel):
     token_type: str
 
 
-oauth2_scheme = OAuth2PasswordBearer(tokenUrl='/api/v1/auth/token', auto_error=False)
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl=f'{root_path}/auth/token', auto_error=False)
 
 
 async def get_optional_user(access_token: str = Depends(oauth2_scheme)) -> User:

nomad/app/routers/datasets.py → nomad/app/v1/routers/datasets.py

@@ -27,13 +27,12 @@ from nomad.utils import strip, create_uuid
 from nomad.datamodel import Dataset as DatasetDefinitionCls
 from nomad.doi import DOI
 
-from nomad.app.routers.auth import get_required_user
-from nomad.app.utils import create_responses
-from nomad.app.models import (
-    pagination_parameters, Pagination, PaginationResponse, Query,
-    HTTPExceptionModel, User, Direction, Owner)
-
+from .auth import get_required_user
 from .entries import _do_exaustive_search
+from ..utils import create_responses
+from ..models import (
+    pagination_parameters, Pagination, PaginationResponse, Query, HTTPExceptionModel,
+    User, Direction, Owner)
 
 
 router = APIRouter()

nomad/app/routers/entries.py → nomad/app/v1/routers/entries.py

@@ -29,9 +29,10 @@ from nomad.utils import strip
 from nomad.archive import (
     query_archive, ArchiveQueryError, compute_required_with_referenced,
     read_partial_archives_from_mongo, filter_archive)
-from nomad.app.utils import create_streamed_zipfile, File, create_responses
-from nomad.app.routers.auth import get_optional_user
-from nomad.app.models import (
+
+from .auth import get_optional_user
+from ..utils import create_streamed_zipfile, File, create_responses
+from ..models import (
     Pagination, WithQuery, MetadataRequired, EntriesMetadataResponse, EntriesMetadata,
     EntryMetadataResponse, query_parameters, metadata_required_parameters, Files, Query,
     pagination_parameters, files_parameters, User, Owner, HTTPExceptionModel, EntriesRaw,

nomad/app/routers/users.py → nomad/app/v1/routers/users.py

@@ -18,11 +18,12 @@
 
 from fastapi import Depends, APIRouter, status
 
-from nomad.app.routers.auth import get_required_user
-from nomad.app.models import User, HTTPExceptionModel
-from nomad.app.utils import create_responses
 from nomad.utils import strip
 
+from .auth import get_required_user
+from ..models import User, HTTPExceptionModel
+from ..utils import create_responses
+
 router = APIRouter()
 default_tag = 'users'
 

nomad/search.py

@@ -31,8 +31,8 @@ from nomad.datamodel.material import Material
 from nomad import config, datamodel, infrastructure, utils
 from nomad.metainfo.search_extension import (  # pylint: disable=unused-import
     search_quantities, metrics, order_default_quantities, groups)
-from nomad.app import models as api_models
-from nomad.app.models import (
+from nomad.app.v1 import models as api_models
+from nomad.app.v1.models import (
     Pagination, PaginationResponse, Query, MetadataRequired, SearchResponse, Aggregation,
     Statistic, StatisticResponse, AggregationOrderType, AggregationResponse, AggregationDataItem)
 

tests/app/conftest.py

@@ -50,7 +50,7 @@ def admin_user_auth(admin_user: User):
 
 @pytest.fixture(scope='session')
 def client():
-    return TestClient(app, base_url='http://testserver/api/v1/')
+    return TestClient(app, base_url='http://testserver/')
 
 
 @pytest.fixture(scope='module')

tests/app/v1/conftest.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pytest
+from fastapi.testclient import TestClient
+
+from nomad.app.main import app
+
+
+@pytest.fixture(scope='session')
+def client():
+    print('###')
+    return TestClient(app, base_url='http://testserver/api/v1/')

tests/app/routers/test_entries.py → tests/app/v1/routers/test_entries.py

@@ -23,7 +23,7 @@ import io
 import json
 
 from nomad.metainfo.search_extension import search_quantities
-from nomad.app.models import AggregateableQuantity, Metric
+from nomad.app.v1.models import AggregateableQuantity, Metric
 
 from tests.utils import assert_at_least