Added some REST api documentation.

docs/api.rst

+API Documentation
+=================
+
+Summary
+-------
+
+.. qrefflask:: nomad.api:app
+  :undoc-static:
+
+API Details
+-----------
+
+.. autoflask:: nomad.api:app
+  :undoc-static:
\ No newline at end of file

docs/conf.py

@@ -44,14 +44,17 @@ extensions = [
     'sphinx.ext.todo',
     'sphinx.ext.coverage',
     'sphinx.ext.ifconfig',
-    'sphinx.ext.napoleon'
+    'sphinx.ext.napoleon',
+    'sphinxcontrib.httpdomain',
+    'sphinxcontrib.autohttp.flask',
+    'sphinxcontrib.autohttp.flaskqref',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['.templates']
 
 source_parsers = {
-   '.md': 'recommonmark.parser.CommonMarkParser',
+    '.md': 'recommonmark.parser.CommonMarkParser',
 }
 
 # The suffix(es) of source filenames.

docs/index.rst

@@ -9,5 +9,6 @@ and infrastructure with a simplyfied and improved architecture.
 
    setup
    introduction
-   modules
+   api
+   reference
    contributing

docs/modules.md → docs/reference.md

-# Modules
+# Reference
 
 ## nomad.config
 ```eval_rst
@@ -26,3 +26,8 @@
 ```eval_rst
 .. automodule:: nomad.processing
 ```
+
+## nomad.data
+```eval_rst
+.. automodule:: nomad.data
+```

nomad/api.py

@@ -5,7 +5,7 @@ from elasticsearch.exceptions import NotFoundError
 
 from nomad import config, files
 from nomad.utils import get_logger
-from nomad.data import Calc, Upload, User, InvalidId, NotAllowedDuringProcessing
+from nomad.data import Calc, Upload, User, InvalidId, NotAllowedDuringProcessing, me
 
 base_path = config.services.api_base_path
 
@@ -17,19 +17,120 @@ CORS(app)
 api = Api(app)
 
 
-# provid a fake user for testing
-me = User.objects(email='me@gmail.com').first()
-if me is None:
-    me = User(email='me@gmail.com', name='Me Meyer')
-    me.save()
+class UploadsRes(Resource):
+    """ Uploads """
+    def get(self):
+        """
+        Get a list of current users uploads.
 
+        .. :quickref: Get a list of current users uploads.
 
-class UploadsRes(Resource):
+        **Example request**:
 
-    def get(self):
+        .. sourcecode:: http
+
+            GET /nomadxt/api/uploads HTTP/1.1
+            Accept: application/json
+
+        **Example response**:
+
+        .. sourcecode:: http
+
+            HTTP/1.1 200 OK
+            Vary: Accept
+            Content-Type: application/json
+
+            [
+                {
+                    "name": "examples_vasp_6.zip",
+                    "upload_id": "5b89469e0d80d40008077dbc",
+                    "presigned_url": "http://minio:9000/uploads/5b89469e0d80d40008077dbc?X-Amz-Algorithm=AWS4-...",
+                    "create_time": "2018-08-31T13:46:06.781000",
+                    "upload_time": "2018-08-31T13:46:07.531000",
+                    "is_stale": false,
+                    "is_ready": true,
+                    "proc": {
+                        "task_names": [
+                            "uploading",
+                            "extracting",
+                            "parse_all",
+                            "cleanup"
+                        ],
+                        "current_task_name": "cleanup",
+                        "status": "SUCCESS",
+                        "errors": [],
+                        "warnings": [],
+                        "upload_id": "5b89469e0d80d40008077dbc",
+                        "upload_hash": "rMB5F-gyHT0KY22eePoTjXibK95S",
+                        "calc_procs": []
+                    }
+                }
+            ]
+
+        :resheader Content-Type: application/json
+        :status 200: uploads successfully provided
+        :returns: list of :class:`nomad.data.Upload`
+        """
         return [upload.json_dict for upload in Upload.user_uploads(me)], 200
 
     def post(self):
+        """
+        Create a new upload. Creating an upload on its own wont do much, but provide
+        a *presigned* upload URL. PUT a file to this URL to do the actual upload and
+        initiate the processing.
+
+        .. :quickref: Create a new upload.
+
+        **Example request**:
+
+        .. sourcecode:: http
+
+            POST /nomadxt/api/uploads HTTP/1.1
+            Accept: application/json
+            Content-Type: application/json
+
+            {
+                name: 'vasp_data.zip'
+            }
+
+        **Example response**:
+
+        .. sourcecode:: http
+
+            HTTP/1.1 200 OK
+            Vary: Accept
+            Content-Type: application/json
+
+            [
+                {
+                    "name": "vasp_data.zip",
+                    "upload_id": "5b89469e0d80d40008077dbc",
+                    "presigned_url": "http://minio:9000/uploads/5b89469e0d80d40008077dbc?X-Amz-Algorithm=AWS4-...",
+                    "create_time": "2018-08-31T13:46:06.781000",
+                    "is_stale": false,
+                    "is_ready": false,
+                    "proc": {
+                        "task_names": [
+                            "uploading",
+                            "extracting",
+                            "parse_all",
+                            "cleanup"
+                        ],
+                        "current_task_name": "uploading",
+                        "status": "PENDING",
+                        "errors": [],
+                        "warnings": [],
+                        "upload_id": "5b89469e0d80d40008077dbc",
+                        "calc_procs": []
+                    }
+                }
+            ]
+        :jsonparam string name: An optional name for the upload.
+        :reqheader Content-Type: application/json
+        :resheader Content-Type: application/json
+        :status 200: upload successfully created
+        :returns: a new instance of :class:`nomad.data.Upload`
+        """
         json_data = request.get_json()
         if json_data is None:
             json_data = {}
@@ -38,7 +139,62 @@ class UploadsRes(Resource):
 
 
 class UploadRes(Resource):
+    """ Uploads """
     def get(self, upload_id):
+        """
+        Get an update for an existing upload. If the upload will be upaded with new data from the
+        processing infrastucture. You can use this endpoint to periodically pull the
+        new processing state until the upload ``is_ready``.
+
+        .. :quickref: Get an update for an existing upload.
+
+        **Example request**:
+
+        .. sourcecode:: http
+
+            GET /nomadxt/api/uploads/5b89469e0d80d40008077dbc HTTP/1.1
+            Accept: application/json
+
+        **Example response**:
+
+        .. sourcecode:: http
+
+            HTTP/1.1 200 OK
+            Vary: Accept
+            Content-Type: application/json
+
+            [
+                {
+                    "name": "vasp_data.zip",
+                    "upload_id": "5b89469e0d80d40008077dbc",
+                    "presigned_url": "http://minio:9000/uploads/5b89469e0d80d40008077dbc?X-Amz-Algorithm=AWS4-...",
+                    "create_time": "2018-08-31T13:46:06.781000",
+                    "upload_time": "2018-08-31T13:46:16.824000",
+                    "is_stale": false,
+                    "is_ready": false,
+                    "proc": {
+                        "task_names": [
+                            "uploading",
+                            "extracting",
+                            "parse_all",
+                            "cleanup"
+                        ],
+                        "current_task_name": "extracting",
+                        "status": "PROGRESS",
+                        "errors": [],
+                        "warnings": [],
+                        "upload_id": "5b89469e0d80d40008077dbc",
+                        "calc_procs": []
+                    }
+                }
+            ]
+        :param string upload_id: the id for the upload
+        :resheader Content-Type: application/json
+        :status 200: upload successfully updated and retrieved
+        :status 400: bad upload id
+        :status 404: upload with id does not exist
+        :returns: the :class:`nomad.data.Upload` instance
+        """
         try:
             return Upload.get(upload_id=upload_id).json_dict, 200
         except InvalidId:
@@ -47,6 +203,26 @@ class UploadRes(Resource):
             abort(404, message='Upload with id %s does not exist.' % upload_id)
 
     def delete(self, upload_id):
+        """
+        Deletes an existing upload. Only ``is_ready`` or ``is_stale`` uploads
+        can be deleted. Deleting an upload in processing is not allowed.
+
+        .. :quickref: Delete an existing upload.
+
+        **Example request**:
+
+        .. sourcecode:: http
+
+            DELETE /nomadxt/api/uploads/5b89469e0d80d40008077dbc HTTP/1.1
+            Accept: application/json
+
+        :param string upload_id: the id for the upload
+        :resheader Content-Type: application/json
+        :status 200: upload successfully deleted
+        :status 400: upload cannot be deleted
+        :status 404: upload with id does not exist
+        :returns: the :class:`nomad.data.Upload` instance with the latest processing state
+        """
         try:
             return Upload.get(upload_id=upload_id).delete().json_dict, 200
         except InvalidId:

nomad/data.py

@@ -17,10 +17,14 @@ This module comprises a set of persistent document classes that hold all user re
 data. These are information about users, their uploads and datasets, the associated
 calculations, and files
 
-..autoclass:: nomad.data.Calc
-..autoclass:: nomad.data.Upload
-..autoclass:: nomad.data.DataSet
-..autoclass:: nomad.data.User
+
+.. autoclass:: Calc
+    :members:
+.. autoclass:: Upload
+    :members:
+.. autoclass:: DataSet
+.. autoclass:: User
+
 """
 
 from typing import List
@@ -386,3 +390,11 @@ class DataSet(Document):
             'calcs'
         ]
     }
+
+# provid a fake user for testing
+me = None
+if 'sphinx' not in sys.modules:
+    me = User.objects(email='me@gmail.com').first()
+    if me is None:
+        me = User(email='me@gmail.com', name='Me Meyer')
+        me.save()

nomad/dependencies.py

@@ -26,6 +26,7 @@ Preparing dependencies
 To make GIT maintained python modules available, we use:
 
 .. autoclass:: PythonGit
+    :members:
 
 
 Dependencies are configured in

nomad/files.py

@@ -36,6 +36,7 @@ authentication hassly, presigned URLs can be created that can be used directly t
 Uploads
 -------
 .. autoclass:: Upload
+    :members:
 
 """
 from typing import Callable, List, Any, Generator, IO, TextIO, cast
@@ -127,7 +128,6 @@ def upload_put_handler(func: Callable[[str], None]) -> Callable[[], None]:
                         'Unhandled bucket event due to unexprected event format',
                         bucket_event_record=event_record)
 
-
     def wrapper(*args, **kwargs) -> None:
         logger.info('Start listening to uploads notifications.')
 

nomad/parsing.py

@@ -30,6 +30,7 @@ For now, we make a few assumption about parsers
 Each parser is defined via an instance of :class:`Parser`.
 
 .. autoclass:: nomad.parsing.Parser
+    :members:
 
 The parser definitions are available via the following two variables.
 
@@ -39,7 +40,9 @@ The parser definitions are available via the following two variables.
 Parsers in NOMAD-coe use a *backend* to create output.
 
 .. autoclass:: nomad.parsing.AbstractParserBackend
+    :members:
 .. autoclass:: nomad.parsing.LocalBackend
+    :members:
 """
 
 from typing import TextIO, Tuple, List, Any, Callable, IO

nomad/processing/__init__.py

@@ -36,8 +36,11 @@ Represent processing state
 
 
 .. autoclass:: ProcPipeline
+    :members:
 .. autoclass:: UploadProc
+    :members:
 .. autoclass:: CalcProc
+    :members:
 
 Initiate processing
 -------------------

nomad/users.py

-# Copyright 2018 Markus Scheidgen
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an"AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-This module comprises a set of persistent document classes that hold all user related
-data. These are information about users, their uploads and datasets, and the
-associations between users and the assets stored in nomad-xt.
-
-..autoclass:: nomad.users.User
-..autoclass:: nomad.users.Upload
-..autoclass:: nomad.users.DataSet
-"""
-
-import sys
-from mongoengine import \
-    Document, EmailField, StringField, BooleanField, DateTimeField, ListField, \
-    DictField, ReferenceField, connect
-
-from nomad import config
-
-# ensure mongo connection
-if 'sphinx' not in sys.modules:
-    connect(db=config.mongo.users_db, host=config.mongo.host)
-
-
-class User(Document):
-    """ Represents users in the database. """
-    email = EmailField(primary=True)
-    name = StringField()
-
-
-class Upload(Document):
-    """
-    Represents uploads in the databases. Provides persistence access to the files storage,
-    and processing system.
-
-    Attributes:
-        file_name: Optional user provided upload name
-        upload_id: The upload id. Generated by the database.
-        in_staging: True if the upload is still in staging and can be edited by the uploader.
-        is_private: True if the upload and its derivitaves are only visible to the uploader.
-        proc: The :class:`nomad.processing.UploadProc` that holds the processing state.
-        created_time: The timestamp this upload was created.
-        upload_time: The timestamp when the system realised the upload.
-        proc_time: The timestamp when the processing realised finished by the system.
-    """
-
-    name = StringField(default=None)
-
-    in_staging = BooleanField(default=True)
-    is_private = BooleanField(default=False)
-
-    presigned_url = StringField()
-    upload_time = DateTimeField()
-    create_time = DateTimeField()
-
-    proc_time = DateTimeField()
-    proc = DictField()
-
-    user = ReferenceField(User, required=True)
-
-    meta = {
-        'indexes': [
-            'proc.upload_hash',
-            'user'
-        ]
-    }
-
-    @property
-    def upload_id(self):
-        return self.id.__str__()
-
-
-class DataSet(Document):
-    name = StringField()
-    description = StringField()
-    doi = StringField()
-
-    user = ReferenceField(User)
-    calcs = ListField(StringField)
-
-    meta = {
-        'indexes': [
-            'user',
-            'doi',
-            'calcs'
-        ]
-    }

requirements.txt

@@ -13,6 +13,7 @@ numpy
 cython>=0.19
 spglib
 gunicorn
+structlog
 sphinx
 recommonmark
-structlog
+sphinxcontrib.httpdomain