Commit a5485fee authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added some REST api documentation.

parent d53a7e5a
API Documentation
=================
Summary
-------
.. qrefflask:: nomad.api:app
:undoc-static:
API Details
-----------
.. autoflask:: nomad.api:app
:undoc-static:
\ No newline at end of file
......@@ -44,14 +44,17 @@ extensions = [
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.ifconfig',
'sphinx.ext.napoleon'
'sphinx.ext.napoleon',
'sphinxcontrib.httpdomain',
'sphinxcontrib.autohttp.flask',
'sphinxcontrib.autohttp.flaskqref',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['.templates']
source_parsers = {
'.md': 'recommonmark.parser.CommonMarkParser',
'.md': 'recommonmark.parser.CommonMarkParser',
}
# The suffix(es) of source filenames.
......
......@@ -9,5 +9,6 @@ and infrastructure with a simplyfied and improved architecture.
setup
introduction
modules
api
reference
contributing
# Modules
# Reference
## nomad.config
```eval_rst
......@@ -26,3 +26,8 @@
```eval_rst
.. automodule:: nomad.processing
```
## nomad.data
```eval_rst
.. automodule:: nomad.data
```
......@@ -5,7 +5,7 @@ from elasticsearch.exceptions import NotFoundError
from nomad import config, files
from nomad.utils import get_logger
from nomad.data import Calc, Upload, User, InvalidId, NotAllowedDuringProcessing
from nomad.data import Calc, Upload, User, InvalidId, NotAllowedDuringProcessing, me
base_path = config.services.api_base_path
......@@ -17,19 +17,120 @@ CORS(app)
api = Api(app)
# provid a fake user for testing
me = User.objects(email='me@gmail.com').first()
if me is None:
me = User(email='me@gmail.com', name='Me Meyer')
me.save()
class UploadsRes(Resource):
""" Uploads """
def get(self):
"""
Get a list of current users uploads.
.. :quickref: Get a list of current users uploads.
class UploadsRes(Resource):
**Example request**:
def get(self):
.. sourcecode:: http
GET /nomadxt/api/uploads HTTP/1.1
Accept: application/json
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
[
{
"name": "examples_vasp_6.zip",
"upload_id": "5b89469e0d80d40008077dbc",
"presigned_url": "http://minio:9000/uploads/5b89469e0d80d40008077dbc?X-Amz-Algorithm=AWS4-...",
"create_time": "2018-08-31T13:46:06.781000",
"upload_time": "2018-08-31T13:46:07.531000",
"is_stale": false,
"is_ready": true,
"proc": {
"task_names": [
"uploading",
"extracting",
"parse_all",
"cleanup"
],
"current_task_name": "cleanup",
"status": "SUCCESS",
"errors": [],
"warnings": [],
"upload_id": "5b89469e0d80d40008077dbc",
"upload_hash": "rMB5F-gyHT0KY22eePoTjXibK95S",
"calc_procs": []
}
}
]
:resheader Content-Type: application/json
:status 200: uploads successfully provided
:returns: list of :class:`nomad.data.Upload`
"""
return [upload.json_dict for upload in Upload.user_uploads(me)], 200
def post(self):
"""
Create a new upload. Creating an upload on its own wont do much, but provide
a *presigned* upload URL. PUT a file to this URL to do the actual upload and
initiate the processing.
.. :quickref: Create a new upload.
**Example request**:
.. sourcecode:: http
POST /nomadxt/api/uploads HTTP/1.1
Accept: application/json
Content-Type: application/json
{
name: 'vasp_data.zip'
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
[
{
"name": "vasp_data.zip",
"upload_id": "5b89469e0d80d40008077dbc",
"presigned_url": "http://minio:9000/uploads/5b89469e0d80d40008077dbc?X-Amz-Algorithm=AWS4-...",
"create_time": "2018-08-31T13:46:06.781000",
"is_stale": false,
"is_ready": false,
"proc": {
"task_names": [
"uploading",
"extracting",
"parse_all",
"cleanup"
],
"current_task_name": "uploading",
"status": "PENDING",
"errors": [],
"warnings": [],
"upload_id": "5b89469e0d80d40008077dbc",
"calc_procs": []
}
}
]
:jsonparam string name: An optional name for the upload.
:reqheader Content-Type: application/json
:resheader Content-Type: application/json
:status 200: upload successfully created
:returns: a new instance of :class:`nomad.data.Upload`
"""
json_data = request.get_json()
if json_data is None:
json_data = {}
......@@ -38,7 +139,62 @@ class UploadsRes(Resource):
class UploadRes(Resource):
""" Uploads """
def get(self, upload_id):
"""
Get an update for an existing upload. If the upload will be upaded with new data from the
processing infrastucture. You can use this endpoint to periodically pull the
new processing state until the upload ``is_ready``.
.. :quickref: Get an update for an existing upload.
**Example request**:
.. sourcecode:: http
GET /nomadxt/api/uploads/5b89469e0d80d40008077dbc HTTP/1.1
Accept: application/json
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
[
{
"name": "vasp_data.zip",
"upload_id": "5b89469e0d80d40008077dbc",
"presigned_url": "http://minio:9000/uploads/5b89469e0d80d40008077dbc?X-Amz-Algorithm=AWS4-...",
"create_time": "2018-08-31T13:46:06.781000",
"upload_time": "2018-08-31T13:46:16.824000",
"is_stale": false,
"is_ready": false,
"proc": {
"task_names": [
"uploading",
"extracting",
"parse_all",
"cleanup"
],
"current_task_name": "extracting",
"status": "PROGRESS",
"errors": [],
"warnings": [],
"upload_id": "5b89469e0d80d40008077dbc",
"calc_procs": []
}
}
]
:param string upload_id: the id for the upload
:resheader Content-Type: application/json
:status 200: upload successfully updated and retrieved
:status 400: bad upload id
:status 404: upload with id does not exist
:returns: the :class:`nomad.data.Upload` instance
"""
try:
return Upload.get(upload_id=upload_id).json_dict, 200
except InvalidId:
......@@ -47,6 +203,26 @@ class UploadRes(Resource):
abort(404, message='Upload with id %s does not exist.' % upload_id)
def delete(self, upload_id):
"""
Deletes an existing upload. Only ``is_ready`` or ``is_stale`` uploads
can be deleted. Deleting an upload in processing is not allowed.
.. :quickref: Delete an existing upload.
**Example request**:
.. sourcecode:: http
DELETE /nomadxt/api/uploads/5b89469e0d80d40008077dbc HTTP/1.1
Accept: application/json
:param string upload_id: the id for the upload
:resheader Content-Type: application/json
:status 200: upload successfully deleted
:status 400: upload cannot be deleted
:status 404: upload with id does not exist
:returns: the :class:`nomad.data.Upload` instance with the latest processing state
"""
try:
return Upload.get(upload_id=upload_id).delete().json_dict, 200
except InvalidId:
......
......@@ -17,10 +17,14 @@ This module comprises a set of persistent document classes that hold all user re
data. These are information about users, their uploads and datasets, the associated
calculations, and files
..autoclass:: nomad.data.Calc
..autoclass:: nomad.data.Upload
..autoclass:: nomad.data.DataSet
..autoclass:: nomad.data.User
.. autoclass:: Calc
:members:
.. autoclass:: Upload
:members:
.. autoclass:: DataSet
.. autoclass:: User
"""
from typing import List
......@@ -386,3 +390,11 @@ class DataSet(Document):
'calcs'
]
}
# provid a fake user for testing
me = None
if 'sphinx' not in sys.modules:
me = User.objects(email='me@gmail.com').first()
if me is None:
me = User(email='me@gmail.com', name='Me Meyer')
me.save()
......@@ -26,6 +26,7 @@ Preparing dependencies
To make GIT maintained python modules available, we use:
.. autoclass:: PythonGit
:members:
Dependencies are configured in
......
......@@ -36,6 +36,7 @@ authentication hassly, presigned URLs can be created that can be used directly t
Uploads
-------
.. autoclass:: Upload
:members:
"""
from typing import Callable, List, Any, Generator, IO, TextIO, cast
......@@ -127,7 +128,6 @@ def upload_put_handler(func: Callable[[str], None]) -> Callable[[], None]:
'Unhandled bucket event due to unexprected event format',
bucket_event_record=event_record)
def wrapper(*args, **kwargs) -> None:
logger.info('Start listening to uploads notifications.')
......
......@@ -30,6 +30,7 @@ For now, we make a few assumption about parsers
Each parser is defined via an instance of :class:`Parser`.
.. autoclass:: nomad.parsing.Parser
:members:
The parser definitions are available via the following two variables.
......@@ -39,7 +40,9 @@ The parser definitions are available via the following two variables.
Parsers in NOMAD-coe use a *backend* to create output.
.. autoclass:: nomad.parsing.AbstractParserBackend
:members:
.. autoclass:: nomad.parsing.LocalBackend
:members:
"""
from typing import TextIO, Tuple, List, Any, Callable, IO
......
......@@ -36,8 +36,11 @@ Represent processing state
.. autoclass:: ProcPipeline
:members:
.. autoclass:: UploadProc
:members:
.. autoclass:: CalcProc
:members:
Initiate processing
-------------------
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
This module comprises a set of persistent document classes that hold all user related
data. These are information about users, their uploads and datasets, and the
associations between users and the assets stored in nomad-xt.
..autoclass:: nomad.users.User
..autoclass:: nomad.users.Upload
..autoclass:: nomad.users.DataSet
"""
import sys
from mongoengine import \
Document, EmailField, StringField, BooleanField, DateTimeField, ListField, \
DictField, ReferenceField, connect
from nomad import config
# ensure mongo connection
if 'sphinx' not in sys.modules:
connect(db=config.mongo.users_db, host=config.mongo.host)
class User(Document):
""" Represents users in the database. """
email = EmailField(primary=True)
name = StringField()
class Upload(Document):
"""
Represents uploads in the databases. Provides persistence access to the files storage,
and processing system.
Attributes:
file_name: Optional user provided upload name
upload_id: The upload id. Generated by the database.
in_staging: True if the upload is still in staging and can be edited by the uploader.
is_private: True if the upload and its derivitaves are only visible to the uploader.
proc: The :class:`nomad.processing.UploadProc` that holds the processing state.
created_time: The timestamp this upload was created.
upload_time: The timestamp when the system realised the upload.
proc_time: The timestamp when the processing realised finished by the system.
"""
name = StringField(default=None)
in_staging = BooleanField(default=True)
is_private = BooleanField(default=False)
presigned_url = StringField()
upload_time = DateTimeField()
create_time = DateTimeField()
proc_time = DateTimeField()
proc = DictField()
user = ReferenceField(User, required=True)
meta = {
'indexes': [
'proc.upload_hash',
'user'
]
}
@property
def upload_id(self):
return self.id.__str__()
class DataSet(Document):
name = StringField()
description = StringField()
doi = StringField()
user = ReferenceField(User)
calcs = ListField(StringField)
meta = {
'indexes': [
'user',
'doi',
'calcs'
]
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment