diff --git a/backend.Dockerfile b/backend.Dockerfile
index f53e1e65a8f2e39a21e873b2c964044651c575a4..d1af26fa0e539ada27f4be961ce96ae8afd06d1d 100644
--- a/backend.Dockerfile
+++ b/backend.Dockerfile
@@ -34,7 +34,6 @@ COPY nomad/dependencies.py nomad/dependencies.py
COPY nomad/config.py nomad/config.py
RUN python nomad/dependencies.py
-
# last stage is used to install the actual code, nomad user, volumes
FROM final
# transfer installed packages from dependency stage
@@ -48,6 +47,10 @@ COPY --from=dependencies /install /install
RUN apt-get update && apt-get install -y make
COPY . /app
WORKDIR /app
+
+# copy the meta-info, since it files are loaded via relative paths. TODO that should change.
+COPY --from=dependencies /install/.dependencies/nomad-meta-info /app/.dependencies/nomad-meta-info
+
RUN pip install -e .
WORKDIR /app/docs
RUN make html
diff --git a/infrastructure/utils.http b/infrastructure/utils.http
new file mode 100644
index 0000000000000000000000000000000000000000..b14e8ec27a1f0dac958d188bd724fa2cc49117b2
--- /dev/null
+++ b/infrastructure/utils.http
@@ -0,0 +1,26 @@
+###
+# Disable threshold based trigger for turning elastic indices read only due to low disc space
+
+PUT http://localhost:9200/_cluster/settings HTTP/1.1
+content-type: application/json
+
+{
+ "transient" : {
+ "cluster.routing.allocation.disk.threshold_enabled" : false
+ }
+}
+
+###
+# Make calcs index writeable after accedental getting read only
+
+PUT http://localhost:9200/calcs/_settings HTTP/1.1
+content-type: application/json
+
+{
+ "index.blocks.read_only_allow_delete": null
+}
+
+###
+# Delete the calc index
+
+DELETE http://localhost:9200/calcs HTTP/1.1
diff --git a/nomad/api.py b/nomad/api.py
index abc0a7ced2a90aa2a3f195f97591fedc6a9b59ad..74ec74f167efb89728265076084aab5fde904dbe 100644
--- a/nomad/api.py
+++ b/nomad/api.py
@@ -1,3 +1,17 @@
+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
from flask import Flask, request, redirect
from flask_restful import Resource, Api, abort
from flask_cors import CORS
diff --git a/nomad/client.py b/nomad/client.py
new file mode 100644
index 0000000000000000000000000000000000000000..2aa7ee938c0d3fbacdc2d9cc094290cd5c249d74
--- /dev/null
+++ b/nomad/client.py
@@ -0,0 +1,73 @@
+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Simple client library for the nomad api that allows to bulk upload files via shell command.
+"""
+
+import os.path
+import subprocess
+import shlex
+import time
+import sys
+import requests
+
+api_base = 'http://localhost/nomadxt/api'
+
+
+def upload_file(file_path, name=None):
+ """
+ Upload a file to nomad.
+
+ Arguments:
+ file_path: Path to the file, absolute or relative to call directory.
+ name: Optional name, default is the file_path's basename
+ """
+ if name is None:
+ name = os.path.basename(file_path)
+
+ upload = requests.post('%s/uploads' % api_base, data={name: name}).json()
+
+ upload_cmd = upload['upload_command']
+ upload_cmd = upload_cmd.replace('your_file', file_path)
+
+ subprocess.call(shlex.split(upload_cmd))
+
+ print('File uploaded')
+
+ while True:
+ upload = requests.get('%s/uploads/%s' % (api_base, upload['upload_id'])).json()
+ status = upload['status']
+ calcs_pagination = upload['calcs'].get('pagination')
+ if calcs_pagination is None:
+ total, successes, failures = 0, 0, 0
+ else:
+ total, successes, failures = (
+ calcs_pagination[key] for key in ('total', 'successes', 'failures'))
+
+ print(
+ 'status: %s; task: %s; parsing: %d/%d/%d' %
+ (status, upload['current_task'], successes, failures, total))
+
+ if status in ('SUCCESS', 'FAILURE'):
+ break
+
+ time.sleep(5)
+
+
+if __name__ == '__main__':
+ if len(sys.argv) > 3 or len(sys.argv) == 1:
+ print('usage is: <client> filte_to_upload [upload_name]')
+ else:
+ upload_file(sys.argv[1], sys.argv[2] if len(sys.argv) == 3 else None)
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index 1fe16d6160c8ebc5c33311b797da9924b1ae2278..fd7f0de64f38d57f4ecb0e792e34862645e0b3b2 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -271,6 +271,19 @@ class Upload(Proc):
with lnr(logger, 'deleting upload'):
super().delete()
+ @classmethod
+ def _external_objects_url(cls, url):
+ """ Replaces the given internal object storage url (minio) with an URL that allows
+ external access.
+ """
+ port_with_colon = ''
+ if config.services.objects_port > 0:
+ port_with_colon = ':%d' % config.services.objects_port
+
+ return url.replace(
+ '%s:%s' % (config.minio.host, config.minio.port),
+ '%s%s%s' % (config.services.objects_host, port_with_colon, config.services.objects_base_path))
+
@classmethod
def create(cls, **kwargs) -> 'Upload':
"""
@@ -279,7 +292,7 @@ class Upload(Proc):
The upload will be already saved to the database.
"""
self = super().create(**kwargs)
- self.presigned_url = files.get_presigned_upload_url(self.upload_id)
+ self.presigned_url = cls._external_objects_url(files.get_presigned_upload_url(self.upload_id))
self.upload_command = files.create_curl_upload_cmd(self.presigned_url, 'your_file')
self._continue_with('uploading')
return self
diff --git a/requirements.txt b/requirements.txt
index cc1afba35b20994cd2b81a21d5cb3ceb28b53a73..ab32483829a26aee302dbada14fd8463d4327309 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,3 +17,4 @@ structlog
sphinx
recommonmark
sphinxcontrib.httpdomain
+drest
\ No newline at end of file
diff --git a/tests/misc.http b/tests/misc.http
index 833644fbf4c9ca78d14e53578a375c796c0bb7cf..e5b29435db5ae6908c4bc28f691aaaba48ff8bdc 100644
--- a/tests/misc.http
+++ b/tests/misc.http
@@ -1,8 +1,8 @@
-GET http://enc-staging-nomad.esc.rzg.mpg.de/nomadxt/api/uploads HTTP/1.1
+GET http://localhost/nomadxt/api/uploads HTTP/1.1
###
-POST http://enc-staging-nomad.esc.rzg.mpg.de/nomadxt/api/uploads HTTP/1.1
+POST http://localhost/nomadxt/api/uploads HTTP/1.1
content-type: application/json
{