From a5485feef5a9469f8fc6261e3627965b88fc7f87 Mon Sep 17 00:00:00 2001
From: Markus Scheidgen <markus.scheidgen@gmail.com>
Date: Fri, 31 Aug 2018 17:04:28 +0200
Subject: [PATCH] Added some REST api documentation.
---
docs/api.rst | 14 +++
docs/conf.py | 7 +-
docs/index.rst | 3 +-
docs/{modules.md => reference.md} | 7 +-
nomad/api.py | 192 ++++++++++++++++++++++++++++--
nomad/data.py | 20 +++-
nomad/dependencies.py | 1 +
nomad/files.py | 2 +-
nomad/parsing.py | 3 +
nomad/processing/__init__.py | 3 +
nomad/users.py | 99 ---------------
requirements.txt | 3 +-
12 files changed, 237 insertions(+), 117 deletions(-)
create mode 100644 docs/api.rst
rename docs/{modules.md => reference.md} (83%)
delete mode 100644 nomad/users.py
diff --git a/docs/api.rst b/docs/api.rst
new file mode 100644
index 0000000000..121ae97a2f
--- /dev/null
+++ b/docs/api.rst
@@ -0,0 +1,14 @@
+API Documentation
+=================
+
+Summary
+-------
+
+.. qrefflask:: nomad.api:app
+ :undoc-static:
+
+API Details
+-----------
+
+.. autoflask:: nomad.api:app
+ :undoc-static:
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index 585bbed46d..1a1a16a4aa 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -44,14 +44,17 @@ extensions = [
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.ifconfig',
- 'sphinx.ext.napoleon'
+ 'sphinx.ext.napoleon',
+ 'sphinxcontrib.httpdomain',
+ 'sphinxcontrib.autohttp.flask',
+ 'sphinxcontrib.autohttp.flaskqref',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['.templates']
source_parsers = {
- '.md': 'recommonmark.parser.CommonMarkParser',
+ '.md': 'recommonmark.parser.CommonMarkParser',
}
# The suffix(es) of source filenames.
diff --git a/docs/index.rst b/docs/index.rst
index d2f0e8205e..5b413eda18 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -9,5 +9,6 @@ and infrastructure with a simplyfied and improved architecture.
setup
introduction
- modules
+ api
+ reference
contributing
diff --git a/docs/modules.md b/docs/reference.md
similarity index 83%
rename from docs/modules.md
rename to docs/reference.md
index 96b684e3b5..d7ff9d0627 100644
--- a/docs/modules.md
+++ b/docs/reference.md
@@ -1,4 +1,4 @@
-# Modules
+# Reference
## nomad.config
```eval_rst
@@ -26,3 +26,8 @@
```eval_rst
.. automodule:: nomad.processing
```
+
+## nomad.data
+```eval_rst
+.. automodule:: nomad.data
+```
diff --git a/nomad/api.py b/nomad/api.py
index 107eab53e7..639a726927 100644
--- a/nomad/api.py
+++ b/nomad/api.py
@@ -5,7 +5,7 @@ from elasticsearch.exceptions import NotFoundError
from nomad import config, files
from nomad.utils import get_logger
-from nomad.data import Calc, Upload, User, InvalidId, NotAllowedDuringProcessing
+from nomad.data import Calc, Upload, User, InvalidId, NotAllowedDuringProcessing, me
base_path = config.services.api_base_path
@@ -17,19 +17,120 @@ CORS(app)
api = Api(app)
-# provid a fake user for testing
-me = User.objects(email='me@gmail.com').first()
-if me is None:
- me = User(email='me@gmail.com', name='Me Meyer')
- me.save()
+class UploadsRes(Resource):
+ """ Uploads """
+ def get(self):
+ """
+ Get a list of current users uploads.
+ .. :quickref: Get a list of current users uploads.
-class UploadsRes(Resource):
+ **Example request**:
- def get(self):
+ .. sourcecode:: http
+
+ GET /nomadxt/api/uploads HTTP/1.1
+ Accept: application/json
+
+ **Example response**:
+
+ .. sourcecode:: http
+
+ HTTP/1.1 200 OK
+ Vary: Accept
+ Content-Type: application/json
+
+ [
+ {
+ "name": "examples_vasp_6.zip",
+ "upload_id": "5b89469e0d80d40008077dbc",
+ "presigned_url": "http://minio:9000/uploads/5b89469e0d80d40008077dbc?X-Amz-Algorithm=AWS4-...",
+ "create_time": "2018-08-31T13:46:06.781000",
+ "upload_time": "2018-08-31T13:46:07.531000",
+ "is_stale": false,
+ "is_ready": true,
+ "proc": {
+ "task_names": [
+ "uploading",
+ "extracting",
+ "parse_all",
+ "cleanup"
+ ],
+ "current_task_name": "cleanup",
+ "status": "SUCCESS",
+ "errors": [],
+ "warnings": [],
+ "upload_id": "5b89469e0d80d40008077dbc",
+ "upload_hash": "rMB5F-gyHT0KY22eePoTjXibK95S",
+ "calc_procs": []
+ }
+ }
+ ]
+
+ :resheader Content-Type: application/json
+ :status 200: uploads successfully provided
+ :returns: list of :class:`nomad.data.Upload`
+ """
return [upload.json_dict for upload in Upload.user_uploads(me)], 200
def post(self):
+ """
+ Create a new upload. Creating an upload on its own wont do much, but provide
+ a *presigned* upload URL. PUT a file to this URL to do the actual upload and
+ initiate the processing.
+
+ .. :quickref: Create a new upload.
+
+ **Example request**:
+
+ .. sourcecode:: http
+
+ POST /nomadxt/api/uploads HTTP/1.1
+ Accept: application/json
+ Content-Type: application/json
+
+ {
+ name: 'vasp_data.zip'
+ }
+
+ **Example response**:
+
+ .. sourcecode:: http
+
+ HTTP/1.1 200 OK
+ Vary: Accept
+ Content-Type: application/json
+
+ [
+ {
+ "name": "vasp_data.zip",
+ "upload_id": "5b89469e0d80d40008077dbc",
+ "presigned_url": "http://minio:9000/uploads/5b89469e0d80d40008077dbc?X-Amz-Algorithm=AWS4-...",
+ "create_time": "2018-08-31T13:46:06.781000",
+ "is_stale": false,
+ "is_ready": false,
+ "proc": {
+ "task_names": [
+ "uploading",
+ "extracting",
+ "parse_all",
+ "cleanup"
+ ],
+ "current_task_name": "uploading",
+ "status": "PENDING",
+ "errors": [],
+ "warnings": [],
+ "upload_id": "5b89469e0d80d40008077dbc",
+ "calc_procs": []
+ }
+ }
+ ]
+ :jsonparam string name: An optional name for the upload.
+ :reqheader Content-Type: application/json
+ :resheader Content-Type: application/json
+ :status 200: upload successfully created
+ :returns: a new instance of :class:`nomad.data.Upload`
+ """
json_data = request.get_json()
if json_data is None:
json_data = {}
@@ -38,7 +139,62 @@ class UploadsRes(Resource):
class UploadRes(Resource):
+ """ Uploads """
def get(self, upload_id):
+ """
+ Get an update for an existing upload. If the upload will be upaded with new data from the
+ processing infrastucture. You can use this endpoint to periodically pull the
+ new processing state until the upload ``is_ready``.
+
+ .. :quickref: Get an update for an existing upload.
+
+ **Example request**:
+
+ .. sourcecode:: http
+
+ GET /nomadxt/api/uploads/5b89469e0d80d40008077dbc HTTP/1.1
+ Accept: application/json
+
+ **Example response**:
+
+ .. sourcecode:: http
+
+ HTTP/1.1 200 OK
+ Vary: Accept
+ Content-Type: application/json
+
+ [
+ {
+ "name": "vasp_data.zip",
+ "upload_id": "5b89469e0d80d40008077dbc",
+ "presigned_url": "http://minio:9000/uploads/5b89469e0d80d40008077dbc?X-Amz-Algorithm=AWS4-...",
+ "create_time": "2018-08-31T13:46:06.781000",
+ "upload_time": "2018-08-31T13:46:16.824000",
+ "is_stale": false,
+ "is_ready": false,
+ "proc": {
+ "task_names": [
+ "uploading",
+ "extracting",
+ "parse_all",
+ "cleanup"
+ ],
+ "current_task_name": "extracting",
+ "status": "PROGRESS",
+ "errors": [],
+ "warnings": [],
+ "upload_id": "5b89469e0d80d40008077dbc",
+ "calc_procs": []
+ }
+ }
+ ]
+ :param string upload_id: the id for the upload
+ :resheader Content-Type: application/json
+ :status 200: upload successfully updated and retrieved
+ :status 400: bad upload id
+ :status 404: upload with id does not exist
+ :returns: the :class:`nomad.data.Upload` instance
+ """
try:
return Upload.get(upload_id=upload_id).json_dict, 200
except InvalidId:
@@ -47,6 +203,26 @@ class UploadRes(Resource):
abort(404, message='Upload with id %s does not exist.' % upload_id)
def delete(self, upload_id):
+ """
+ Deletes an existing upload. Only ``is_ready`` or ``is_stale`` uploads
+ can be deleted. Deleting an upload in processing is not allowed.
+
+ .. :quickref: Delete an existing upload.
+
+ **Example request**:
+
+ .. sourcecode:: http
+
+ DELETE /nomadxt/api/uploads/5b89469e0d80d40008077dbc HTTP/1.1
+ Accept: application/json
+
+ :param string upload_id: the id for the upload
+ :resheader Content-Type: application/json
+ :status 200: upload successfully deleted
+ :status 400: upload cannot be deleted
+ :status 404: upload with id does not exist
+ :returns: the :class:`nomad.data.Upload` instance with the latest processing state
+ """
try:
return Upload.get(upload_id=upload_id).delete().json_dict, 200
except InvalidId:
diff --git a/nomad/data.py b/nomad/data.py
index 54309d6edc..b73e647460 100644
--- a/nomad/data.py
+++ b/nomad/data.py
@@ -17,10 +17,14 @@ This module comprises a set of persistent document classes that hold all user re
data. These are information about users, their uploads and datasets, the associated
calculations, and files
-..autoclass:: nomad.data.Calc
-..autoclass:: nomad.data.Upload
-..autoclass:: nomad.data.DataSet
-..autoclass:: nomad.data.User
+
+.. autoclass:: Calc
+ :members:
+.. autoclass:: Upload
+ :members:
+.. autoclass:: DataSet
+.. autoclass:: User
+
"""
from typing import List
@@ -386,3 +390,11 @@ class DataSet(Document):
'calcs'
]
}
+
+# provid a fake user for testing
+me = None
+if 'sphinx' not in sys.modules:
+ me = User.objects(email='me@gmail.com').first()
+ if me is None:
+ me = User(email='me@gmail.com', name='Me Meyer')
+ me.save()
diff --git a/nomad/dependencies.py b/nomad/dependencies.py
index 052cc40e0d..e7a4e7c06b 100644
--- a/nomad/dependencies.py
+++ b/nomad/dependencies.py
@@ -26,6 +26,7 @@ Preparing dependencies
To make GIT maintained python modules available, we use:
.. autoclass:: PythonGit
+ :members:
Dependencies are configured in
diff --git a/nomad/files.py b/nomad/files.py
index b1cdd1c3ab..189e519ad5 100644
--- a/nomad/files.py
+++ b/nomad/files.py
@@ -36,6 +36,7 @@ authentication hassly, presigned URLs can be created that can be used directly t
Uploads
-------
.. autoclass:: Upload
+ :members:
"""
from typing import Callable, List, Any, Generator, IO, TextIO, cast
@@ -127,7 +128,6 @@ def upload_put_handler(func: Callable[[str], None]) -> Callable[[], None]:
'Unhandled bucket event due to unexprected event format',
bucket_event_record=event_record)
-
def wrapper(*args, **kwargs) -> None:
logger.info('Start listening to uploads notifications.')
diff --git a/nomad/parsing.py b/nomad/parsing.py
index 2f3a1f9daf..1b9af32efb 100644
--- a/nomad/parsing.py
+++ b/nomad/parsing.py
@@ -30,6 +30,7 @@ For now, we make a few assumption about parsers
Each parser is defined via an instance of :class:`Parser`.
.. autoclass:: nomad.parsing.Parser
+ :members:
The parser definitions are available via the following two variables.
@@ -39,7 +40,9 @@ The parser definitions are available via the following two variables.
Parsers in NOMAD-coe use a *backend* to create output.
.. autoclass:: nomad.parsing.AbstractParserBackend
+ :members:
.. autoclass:: nomad.parsing.LocalBackend
+ :members:
"""
from typing import TextIO, Tuple, List, Any, Callable, IO
diff --git a/nomad/processing/__init__.py b/nomad/processing/__init__.py
index c4104181ae..27448b50be 100644
--- a/nomad/processing/__init__.py
+++ b/nomad/processing/__init__.py
@@ -36,8 +36,11 @@ Represent processing state
.. autoclass:: ProcPipeline
+ :members:
.. autoclass:: UploadProc
+ :members:
.. autoclass:: CalcProc
+ :members:
Initiate processing
-------------------
diff --git a/nomad/users.py b/nomad/users.py
deleted file mode 100644
index b662eb2716..0000000000
--- a/nomad/users.py
+++ /dev/null
@@ -1,99 +0,0 @@
-# Copyright 2018 Markus Scheidgen
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an"AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-This module comprises a set of persistent document classes that hold all user related
-data. These are information about users, their uploads and datasets, and the
-associations between users and the assets stored in nomad-xt.
-
-..autoclass:: nomad.users.User
-..autoclass:: nomad.users.Upload
-..autoclass:: nomad.users.DataSet
-"""
-
-import sys
-from mongoengine import \
- Document, EmailField, StringField, BooleanField, DateTimeField, ListField, \
- DictField, ReferenceField, connect
-
-from nomad import config
-
-# ensure mongo connection
-if 'sphinx' not in sys.modules:
- connect(db=config.mongo.users_db, host=config.mongo.host)
-
-
-class User(Document):
- """ Represents users in the database. """
- email = EmailField(primary=True)
- name = StringField()
-
-
-class Upload(Document):
- """
- Represents uploads in the databases. Provides persistence access to the files storage,
- and processing system.
-
- Attributes:
- file_name: Optional user provided upload name
- upload_id: The upload id. Generated by the database.
- in_staging: True if the upload is still in staging and can be edited by the uploader.
- is_private: True if the upload and its derivitaves are only visible to the uploader.
- proc: The :class:`nomad.processing.UploadProc` that holds the processing state.
- created_time: The timestamp this upload was created.
- upload_time: The timestamp when the system realised the upload.
- proc_time: The timestamp when the processing realised finished by the system.
- """
-
- name = StringField(default=None)
-
- in_staging = BooleanField(default=True)
- is_private = BooleanField(default=False)
-
- presigned_url = StringField()
- upload_time = DateTimeField()
- create_time = DateTimeField()
-
- proc_time = DateTimeField()
- proc = DictField()
-
- user = ReferenceField(User, required=True)
-
- meta = {
- 'indexes': [
- 'proc.upload_hash',
- 'user'
- ]
- }
-
- @property
- def upload_id(self):
- return self.id.__str__()
-
-
-class DataSet(Document):
- name = StringField()
- description = StringField()
- doi = StringField()
-
- user = ReferenceField(User)
- calcs = ListField(StringField)
-
- meta = {
- 'indexes': [
- 'user',
- 'doi',
- 'calcs'
- ]
- }
diff --git a/requirements.txt b/requirements.txt
index 788b21caae..cc1afba35b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,6 +13,7 @@ numpy
cython>=0.19
spglib
gunicorn
+structlog
sphinx
recommonmark
-structlog
+sphinxcontrib.httpdomain
--
GitLab