Refactored uploads API and its tests.

nomad/api/__init__.py

@@ -29,7 +29,7 @@ There is a separate documentation for the API endpoints from a client perspectiv
 .. automodule:: nomad.api.admin
 """
 from .app import app
-from . import auth, admin, upload, repository, archive, raw, upload_v2
+from . import auth, admin, upload, repository, archive, raw
 
 
 @app.before_first_request

nomad/api/app.py

@@ -33,9 +33,20 @@ app = Flask(
     static_folder=os.path.abspath(os.path.join(os.path.dirname(__file__), '../../docs/.build/html')))
 """ The Flask app that serves all APIs. """
 
+app.config.setdefault('RESTPLUS_MASK_HEADER', False)
+app.config.setdefault('RESTPLUS_MASK_SWAGGER', False)
+
 CORS(app)
 
-api = Api(app)
+authorizations = {
+    'HTTP Basic': {
+        'type': 'basic'
+    }
+}
+
+api = Api(
+    app, version='1.0', title='nomad@FAIRDI API', authorizations=authorizations,
+    description='Official API for nomad@FAIRDI services.')
 """ Provides the flask restful api instance """
 
 

nomad/api/auth.py

@@ -42,7 +42,7 @@ from flask_httpauth import HTTPBasicAuth
 from nomad import config
 from nomad.coe_repo import User
 
-from .app import app, base_path
+from .app import app, api, base_path
 
 app.config['SECRET_KEY'] = config.services.api_secret
 auth = HTTPBasicAuth()
@@ -92,6 +92,8 @@ def login_really_required(func):
     A decorator for API endpoint implementations that forces user authentication on
     endpoints.
     """
+    @api.response(401, 'Not Authorized')
+    @api.doc(security=['HTTP Basic'])
     @login_if_available
     def wrapper(*args, **kwargs):
         if g.user is None:

nomad/processing/base.py

@@ -87,7 +87,6 @@ class Proc(Document, metaclass=ProcMetaclass):
 
     Processing state will be persistet at appropriate
     times and must not be persistet manually. All attributes are stored to mongodb.
-    The class allows to render into a JSON serializable dict via :attr:`json_dict`.
 
     Possible processing states are PENDING, RUNNING, FAILURE, and SUCCESS.
 
@@ -254,22 +253,6 @@ class Proc(Document, metaclass=ProcMetaclass):
             time.sleep(interval)
             self.reload()
 
-    @property
-    def json_dict(self) -> dict:
-        """ A json serializable dictionary representation. """
-        data = {
-            'tasks': getattr(self.__class__, 'tasks'),
-            'current_task': self.current_task,
-            'status': self.status,
-            'completed': self.completed,
-            'errors': self.errors,
-            'warnings': self.warnings,
-            'create_time': self.create_time.isoformat() if self.create_time is not None else None,
-            'complete_time': self.complete_time.isoformat() if self.complete_time is not None else None,
-            '_async_status': self._async_status
-        }
-        return {key: value for key, value in data.items() if value is not None}
-
 
 class InvalidChordUsage(Exception): pass
 

nomad/processing/data.py

@@ -25,7 +25,6 @@ calculations, and files
 """
 
 from typing import List, Any, ContextManager, Tuple, Generator
-from datetime import datetime
 from elasticsearch.exceptions import NotFoundError
 from mongoengine import StringField, BooleanField, DateTimeField, DictField, IntField
 import logging
@@ -154,18 +153,6 @@ class Calc(Proc):
 
         return wrap_logger(logger, processors=[save_to_calc_log])
 
-    @property
-    def json_dict(self):
-        """ A json serializable dictionary representation. """
-        data = {
-            'archive_id': self.archive_id,
-            'mainfile': self.mainfile,
-            'upload_id': self.upload_id,
-            'parser': self.parser
-        }
-        data.update(super().json_dict)
-        return {key: value for key, value in data.items() if value is not None}
-
     @process
     def process(self):
         logger = self.get_logger()
@@ -372,7 +359,7 @@ class Upload(Chord):
     def delete(self):
         logger = self.get_logger(task='delete')
 
-        if not (self.completed or self.is_stale or self.current_task == 'uploading'):
+        if not (self.completed or self.current_task == 'uploading'):
             raise NotAllowedDuringProcessing()
 
         with lnr(logger, 'delete upload file'):
@@ -433,37 +420,17 @@ class Upload(Chord):
 
         return self
 
-    @property
-    def is_stale(self) -> bool:
-        if self.current_task == 'uploading' and self.upload_time is None:
-            return (datetime.now() - self.create_time).days > 1
-        else:
-            return False
-
     def unstage(self):
         self.get_logger().info('unstage')
+
+        if not (self.completed or self.current_task == 'uploading'):
+            raise NotAllowedDuringProcessing()
+
         self.in_staging = False
         RepoCalc.unstage(upload_id=self.upload_id)
         coe_repo.add_upload(self, restricted=False)  # TODO allow users to choose restricted
         self.save()
 
-    @property
-    def json_dict(self) -> dict:
-        """ A json serializable dictionary representation. """
-        data = {
-            'name': self.name,
-            'local_path': self.local_path,
-            'additional_metadata': self.additional_metadata,
-            'upload_id': self.upload_id,
-            'upload_hash': self.upload_hash,
-            'upload_url': self.upload_url,
-            'upload_command': self.upload_command,
-            'upload_time': self.upload_time.isoformat() if self.upload_time is not None else None,
-            'is_stale': self.is_stale,
-        }
-        data.update(super().json_dict)
-        return {key: value for key, value in data.items() if value is not None}
-
     @process
     def process(self):
         self.extracting()

nomad/uploads.py

@@ -37,7 +37,7 @@ almost readonly (beside metadata) storage.
 """
 
 from abc import ABCMeta
-from typing import IO, Generator, Dict, Iterator, Iterable, Callable
+from typing import IO, Generator, Dict, Iterator, Iterable
 from filelock import Timeout, FileLock
 import ujson
 import os.path

nomad/utils.py

@@ -52,6 +52,7 @@ from nomad import config
 default_hash_len = 28
 """ Length of hashes and hash-based ids (e.g. calc, upload) in nomad. """
 
+
 def sanitize_logevent(event: str) -> str:
     """
     Prepares a log event or message for analysis in elastic stack. It removes numbers,

tests/conftest.py

@@ -145,7 +145,7 @@ def mocksearch(monkeypatch):
         if upload_id in uploads_by_id:
             for calc in uploads_by_id[upload_id]:
                 del(by_archive_id[calc.archive_id])
-            upload_hash = next(uploads_by_id[upload_id]).upload_hash
+            upload_hash = uploads_by_id[upload_id][0].upload_hash
             del(uploads_by_id[upload_id])
             del(uploads_by_hash[upload_hash])
 

tests/misc.http

@@ -8,3 +8,8 @@ content-type: application/json
 {
     "name": "aims-example-full"
 }
+###
+
+GET http://localhost:8000/nomad/api/v2/uploads/ HTTP/1.1
+Authorization: Basic bGVvbmFyZC5ob2ZzdGFkdGVyQG5vbWFkLWZhaXJkaS50ZXN0cy5kZTo=
+###