Initial checkin of basic project skeleton. #8

.gitignore

+.DS_Store
+.*env/
+.pyenv*/
+.pytest/
+.python-version
+.ipynb_checkpoints/
+.python-version
+.coverage_html/
+__pycache__
+.mypy_cache
+*.pyc
+*.egg-info/
+/data/
+.volumes/
+.pytest_cache/
+.coverage*
+htmlcov
+try.http
+project/
+test_*/
+local/
+target/
+*.swp
+*.vscode
+.vscode/
+vscode/
+nomad.yaml
+gunicorn.log.conf
+gunicorn.conf
+build/
+dist/
+setup.json
+parser.osio.log
+gui/src/metainfo.json
+gui/src/searchQuantities.json
+gui/src/toolkitMetadata.json
+gui/src/unitsData.js
+examples/workdir/
+gunicorn.log.conf
+nomad/gitinfo.py
+*/node_modules/
+

.gitlab-ci.yml

+image: python:3.7
+
+stages:
+  - test
+
+linting:
+  stage: test
+  script:
+    - pip install -e .
+    - python -m pycodestyle --ignore=E501,E701,E731 north tests
+    - python -m pylint north tests
+    - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional north tests
+
+tests:
+  stage: test
+  script:
+    - pip install -e .
+    - python -m pytest --cov=north -sv tests

AUTHORS

+Markus Scheidgen <markus.scheidgen@gmail.com>
+TODO developers add yourself

MANIFEST.in

+recursive-include dependencies/optimade-python-tools *.txt *.g *.py *.ini *.json
+recursive-include nomad *.json *.j2 *.md *.yaml
+include nomad/units/*.txt
+include README.md
+include LICENSE.txt
+include requirements.txt
+include auto_complete_install.sh
+include setup.json
+recursive-include nomad/app/static *.css *.ico *.html *.json *.js *.map *.txt *.svg *.png
\ No newline at end of file

README.md

+[![pipeline status](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-remote-tools-hub/badges/main/pipeline.svg)](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-remote-tools-hub/commits/main)
+[![coverage report](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-remote-tools-hub/badges/main/coverage.svg)](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-remote-tools-hub/commits/main)
+
 # NOMAD remote tools hub (north)
 
 Lets you run containarized tools remotly.
 
+## Getting started
+
+Clone the project
+
+```sh
+git clone git@gitlab.mpcdf.mpg.de:nomad-lab/nomad-remote-tools-hub.git
+cd nomad-remote-tools-hub
+```
+
+Optionaly, checkout the desired branch (e.g. develop) and create a feature branch
+```
+git checkout develop
+git checkout -b my-feature
+```
+
+Create a virtual environment based on Python 3 (>3.7).
+```sh
+pip install virtualenv
+virtualenv -p `which python3` .pyenv
+source .pyenv/bin/activate
+```
+
+Install the nomad-remote-tools-hub package.
+```sh
+pip install -e .
+```
+
+Run the app with uvivcorn:
+```sh
+uvicorn north.app.main:app
+```
+
+Run the tests with pytest:
+```sh
+pytest -svx test
+```
+
+We recomment using vs-code. Here are vs-code settings that match the CI/CD linting:
+```json
+{
+    "python.pythonPath": ".pyenv/bin/python",
+    "editor.rulers": [90],
+    "editor.renderWhitespace": "all",
+    "editor.tabSize": 4,
+    "files.trimTrailingWhitespace": true,
+    "python.linting.pycodestylePath": "pycodestyle",
+    "python.linting.pycodestyleEnabled": true,
+    "python.linting.pycodestyleArgs": ["--ignore=E501,E701,E731"],
+    "python.linting.mypyEnabled": true,
+    "python.linting.pylintEnabled": true,
+}
+```
+
+## Project structure
+
+- `north` - The Python code
+- `north/app` - The [FastAPI](https://fastapi.tiangolo.com/) application that runs the north app
+- `north/config` - All applications settings
+- `tests` - The [pytest](https://docs.pytest.org/) tests
+- `setup.py` - Install the package with pip
+- `docker` - All the docker files, scripts for creating/managing images, documentation

docker/README.md

+**TODO** This directory should contain the docker files and image documentation
+- one folder per "tool"
+- also for images necessary for tests (e.g. nginx with static rev.proxy rules)
+- scripts to create images and push them to the projects registry
\ No newline at end of file

north/__init__.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from .config import config

north/app/main.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from fastapi import FastAPI
+from fastapi.responses import HTMLResponse
+from starlette.exceptions import HTTPException as StarletteHTTPException
+from fastapi.exception_handlers import http_exception_handler as default_http_exception_handler
+
+from north import config
+
+from .routes import instances
+from .routes import tools
+
+app = FastAPI(
+    title='NOMAD remote tools hub API',
+    version=config.version,
+    description=(
+        'This is the API for the NOMAD remote tools hub. It allows to run dockerized '
+        'tools remotely.'))
+
+app.include_router(tools.router, prefix='/tools')
+app.include_router(instances.router, prefix='/instances')
+
+
+# A default 404 response with a link to the API dashboard for convinience
+@app.exception_handler(StarletteHTTPException)
+async def http_exception_handler(request, exc):
+    if exc.status_code != 404:
+        return await default_http_exception_handler(request, exc)
+
+    try:
+        accept = request.headers['accept']
+    except Exception:
+        accept = None
+
+    if accept is not None and 'html' in accept:
+        return HTMLResponse(status_code=404, content='''
+            <html>
+                <head><title>NOMAD remote tools hub app</title></head>
+                <body>
+                    <h1>NOMAD remote tools hub app</h1>
+                    <h2>apis</h2>
+                    <a href="/docs">OpenAPI dashboard</a>
+                </body>
+            </html>
+            ''')

north/app/models.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import List, Dict, Optional
+from pydantic import BaseModel, validator, Field
+
+# TODO This exemplifies pydantic models a little bit. But, this is just for demonstration.
+# The models completed/rewritten and most descriptions are missing.
+
+
+class ToolModel(BaseModel):
+    ''' Model that describes an available tool. '''
+    name: str
+    description: str
+    docker_image: str = Field(..., description='The docker image for this tool.')
+
+
+all_tools: List[ToolModel] = [
+    ToolModel(
+        name='jupyter',
+        description='Basic jupyter run with an empty notebook or on given notebook file.',
+        docker_image='TODO'
+    ),
+    ToolModel(
+        name='hyperspy',
+        description='Run hyperspy on a arbitrary .hdf5 file.',
+        docker_image='TODO'
+    )
+]
+
+tools_map: Dict[str, ToolModel] = {tool.name: tool for tool in all_tools}
+
+
+class InstanceModel(BaseModel):
+    name: str
+    docker_id: Optional[str]
+    tool_name: str
+
+    @validator('tool_name')
+    def validate_tool_name(cls, tool_name):  # pylint: disable=no-self-argument
+        assert tool_name in tools_map, 'Tool must exist.'
+        return tool_name

north/app/routes/instances.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import List
+from fastapi import APIRouter
+
+from north.app.models import InstanceModel
+
+router = APIRouter()
+router_tag = 'instances'
+
+
+@router.get(
+    '/',
+    tags=[router_tag],
+    response_model=List[InstanceModel],
+    response_model_exclude_unset=True,
+    response_model_exclude_none=True)
+async def get_instances():
+    ''' Get a list of all existing tool instances. '''
+    return []
+
+
+@router.post(
+    '/',
+    tags=[router_tag],
+    response_model=InstanceModel,
+    response_model_exclude_unset=True,
+    response_model_exclude_none=True)
+async def post_instances(instance: InstanceModel):
+    ''' Create a new tool instance. '''
+    return instance

north/app/routes/tools.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import List
+from fastapi import APIRouter
+
+from north.app.models import ToolModel, all_tools
+
+router = APIRouter()
+router_tag = 'tools'
+
+
+@router.get(
+    '/',
+    tags=[router_tag],
+    response_model=List[ToolModel],
+    response_model_exclude_unset=True,
+    response_model_exclude_none=True)
+async def get_tools():
+    return all_tools

north/config.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+'''
+This config file is based on pydantic's
+[settings management](https://pydantic-docs.helpmanual.io/usage/settings/).
+'''
+
+from typing import Dict, Any
+from pydantic import Field, BaseSettings
+import yaml
+import os.path
+import os
+
+
+class NorthConfig(BaseSettings):
+    secret: str = Field(
+        'this is a secret',
+        description='The secret for generating JWT tokens and other cryptographic material.')
+
+    version: str = Field(
+        'v0.1.0', description='The current application version according to semantic versioning conventions')
+
+    class Config:
+        env_prefix = 'north_'
+        case_sensitive = False
+
+        @classmethod
+        def customise_sources(
+                cls,
+                init_settings,
+                env_settings,
+                file_secret_settings):
+
+            return (
+                init_settings,
+                env_settings,
+                yaml_config_settings_source, file_secret_settings)
+
+
+def yaml_config_settings_source(settings: BaseSettings) -> Dict[str, Any]:
+    if not os.path.exists('north.yaml'):
+        return {}
+
+    try:
+        with open('north.yaml') as f:
+            data = yaml.load(f, Loader=yaml.FullLoader)
+        if data is None:
+            return {}
+        return data
+    except yaml.YAMLError as e:
+        raise e
+
+
+config = NorthConfig()

requirements.txt

+pyyaml
+fastapi
+requests
+setuptools
+mypy
+pylint
+pycodestyle
+pytest
+pytest-cov
+types-PyYAML
+types-requests

setup.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from setuptools import setup, find_packages
+
+
+with open('requirements.txt') as f:
+    requirements = f.read().splitlines()
+
+
+def main():
+    setup(
+        name='nomad-remote-tools-hub',
+        version='0.1.0',
+        description='NOMAD remote tools hub allows to run containarized tools remotely',
+        author='The NOMAD Authors',
+        license='APACHE 2.0',
+        packages=find_packages(exclude=['tests']),
+        install_requires=requirements)
+
+
+if __name__ == '__main__':
+    main()

tests/__init__.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

tests/app/__init__.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

tests/app/routes/__init__.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

tests/app/routes/test_instance.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pytest
+
+
+def test_get_instances(api):
+    response = api.get('instances')
+    assert response.status_code == 200, response.text
+    assert len(response.json()) == 0
+
+
+@pytest.mark.parametrize('request_json, status_code', [
+    pytest.param({'name': 'myinstance', 'tool_name': 'jupyter'}, 200, id='ok'),
+    pytest.param({'name': 'myinstance', 'tool_name': 'doesnotexist'}, 422, id='tool-does-not-exist'),
+    pytest.param({'tool_name': 'jupyter'}, 422, id='name-is-missing')
+])
+def test_post_instances(api, request_json, status_code):
+    response = api.post('instances/', json=request_json)
+    assert response.status_code == status_code, response.text