Commit 03d405b0 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored uploads API and its tests.

parent c9a5d798
......@@ -29,7 +29,7 @@ There is a separate documentation for the API endpoints from a client perspectiv
.. automodule:: nomad.api.admin
"""
from .app import app
from . import auth, admin, upload, repository, archive, raw, upload_v2
from . import auth, admin, upload, repository, archive, raw
@app.before_first_request
......
......@@ -33,9 +33,20 @@ app = Flask(
static_folder=os.path.abspath(os.path.join(os.path.dirname(__file__), '../../docs/.build/html')))
""" The Flask app that serves all APIs. """
app.config.setdefault('RESTPLUS_MASK_HEADER', False)
app.config.setdefault('RESTPLUS_MASK_SWAGGER', False)
CORS(app)
api = Api(app)
authorizations = {
'HTTP Basic': {
'type': 'basic'
}
}
api = Api(
app, version='1.0', title='nomad@FAIRDI API', authorizations=authorizations,
description='Official API for nomad@FAIRDI services.')
""" Provides the flask restful api instance """
......
......@@ -42,7 +42,7 @@ from flask_httpauth import HTTPBasicAuth
from nomad import config
from nomad.coe_repo import User
from .app import app, base_path
from .app import app, api, base_path
app.config['SECRET_KEY'] = config.services.api_secret
auth = HTTPBasicAuth()
......@@ -92,6 +92,8 @@ def login_really_required(func):
A decorator for API endpoint implementations that forces user authentication on
endpoints.
"""
@api.response(401, 'Not Authorized')
@api.doc(security=['HTTP Basic'])
@login_if_available
def wrapper(*args, **kwargs):
if g.user is None:
......
This diff is collapsed.
......@@ -87,7 +87,6 @@ class Proc(Document, metaclass=ProcMetaclass):
Processing state will be persistet at appropriate
times and must not be persistet manually. All attributes are stored to mongodb.
The class allows to render into a JSON serializable dict via :attr:`json_dict`.
Possible processing states are PENDING, RUNNING, FAILURE, and SUCCESS.
......@@ -254,22 +253,6 @@ class Proc(Document, metaclass=ProcMetaclass):
time.sleep(interval)
self.reload()
@property
def json_dict(self) -> dict:
""" A json serializable dictionary representation. """
data = {
'tasks': getattr(self.__class__, 'tasks'),
'current_task': self.current_task,
'status': self.status,
'completed': self.completed,
'errors': self.errors,
'warnings': self.warnings,
'create_time': self.create_time.isoformat() if self.create_time is not None else None,
'complete_time': self.complete_time.isoformat() if self.complete_time is not None else None,
'_async_status': self._async_status
}
return {key: value for key, value in data.items() if value is not None}
class InvalidChordUsage(Exception): pass
......
......@@ -25,7 +25,6 @@ calculations, and files
"""
from typing import List, Any, ContextManager, Tuple, Generator
from datetime import datetime
from elasticsearch.exceptions import NotFoundError
from mongoengine import StringField, BooleanField, DateTimeField, DictField, IntField
import logging
......@@ -154,18 +153,6 @@ class Calc(Proc):
return wrap_logger(logger, processors=[save_to_calc_log])
@property
def json_dict(self):
""" A json serializable dictionary representation. """
data = {
'archive_id': self.archive_id,
'mainfile': self.mainfile,
'upload_id': self.upload_id,
'parser': self.parser
}
data.update(super().json_dict)
return {key: value for key, value in data.items() if value is not None}
@process
def process(self):
logger = self.get_logger()
......@@ -372,7 +359,7 @@ class Upload(Chord):
def delete(self):
logger = self.get_logger(task='delete')
if not (self.completed or self.is_stale or self.current_task == 'uploading'):
if not (self.completed or self.current_task == 'uploading'):
raise NotAllowedDuringProcessing()
with lnr(logger, 'delete upload file'):
......@@ -433,37 +420,17 @@ class Upload(Chord):
return self
@property
def is_stale(self) -> bool:
if self.current_task == 'uploading' and self.upload_time is None:
return (datetime.now() - self.create_time).days > 1
else:
return False
def unstage(self):
self.get_logger().info('unstage')
if not (self.completed or self.current_task == 'uploading'):
raise NotAllowedDuringProcessing()
self.in_staging = False
RepoCalc.unstage(upload_id=self.upload_id)
coe_repo.add_upload(self, restricted=False) # TODO allow users to choose restricted
self.save()
@property
def json_dict(self) -> dict:
""" A json serializable dictionary representation. """
data = {
'name': self.name,
'local_path': self.local_path,
'additional_metadata': self.additional_metadata,
'upload_id': self.upload_id,
'upload_hash': self.upload_hash,
'upload_url': self.upload_url,
'upload_command': self.upload_command,
'upload_time': self.upload_time.isoformat() if self.upload_time is not None else None,
'is_stale': self.is_stale,
}
data.update(super().json_dict)
return {key: value for key, value in data.items() if value is not None}
@process
def process(self):
self.extracting()
......
......@@ -37,7 +37,7 @@ almost readonly (beside metadata) storage.
"""
from abc import ABCMeta
from typing import IO, Generator, Dict, Iterator, Iterable, Callable
from typing import IO, Generator, Dict, Iterator, Iterable
from filelock import Timeout, FileLock
import ujson
import os.path
......
......@@ -52,6 +52,7 @@ from nomad import config
default_hash_len = 28
""" Length of hashes and hash-based ids (e.g. calc, upload) in nomad. """
def sanitize_logevent(event: str) -> str:
"""
Prepares a log event or message for analysis in elastic stack. It removes numbers,
......
......@@ -145,7 +145,7 @@ def mocksearch(monkeypatch):
if upload_id in uploads_by_id:
for calc in uploads_by_id[upload_id]:
del(by_archive_id[calc.archive_id])
upload_hash = next(uploads_by_id[upload_id]).upload_hash
upload_hash = uploads_by_id[upload_id][0].upload_hash
del(uploads_by_id[upload_id])
del(uploads_by_hash[upload_hash])
......
......@@ -8,3 +8,8 @@ content-type: application/json
{
"name": "aims-example-full"
}
###
GET http://localhost:8000/nomad/api/v2/uploads/ HTTP/1.1
Authorization: Basic bGVvbmFyZC5ob2ZzdGFkdGVyQG5vbWFkLWZhaXJkaS50ZXN0cy5kZTo=
###
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment