Added simple client for uploading. Fixed external objects urls, some utils for elastic troubles.

backend.Dockerfile

@@ -34,7 +34,6 @@ COPY nomad/dependencies.py nomad/dependencies.py
 COPY nomad/config.py nomad/config.py
 RUN python nomad/dependencies.py
 
-
 # last stage is used to install the actual code, nomad user, volumes
 FROM final
 # transfer installed packages from dependency stage
@@ -48,6 +47,10 @@ COPY --from=dependencies /install /install
 RUN apt-get update && apt-get install -y make
 COPY . /app
 WORKDIR /app
+
+# copy the meta-info, since it files are loaded via relative paths. TODO that should change.
+COPY --from=dependencies /install/.dependencies/nomad-meta-info /app/.dependencies/nomad-meta-info
+
 RUN pip install -e .
 WORKDIR /app/docs
 RUN make html

infrastructure/utils.http

+###
+# Disable threshold based trigger for turning elastic indices read only due to low disc space
+
+PUT http://localhost:9200/_cluster/settings HTTP/1.1
+content-type: application/json
+
+{
+    "transient" : {
+        "cluster.routing.allocation.disk.threshold_enabled" : false
+    }
+}
+
+###
+# Make calcs index writeable after accedental getting read only
+
+PUT http://localhost:9200/calcs/_settings HTTP/1.1
+content-type: application/json
+
+{
+    "index.blocks.read_only_allow_delete": null
+}
+
+###
+# Delete the calc index
+
+DELETE http://localhost:9200/calcs HTTP/1.1

nomad/api.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from flask import Flask, request, redirect
 from flask_restful import Resource, Api, abort
 from flask_cors import CORS

nomad/client.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Simple client library for the nomad api that allows to bulk upload files via shell command.
+"""
+
+import os.path
+import subprocess
+import shlex
+import time
+import sys
+import requests
+
+api_base = 'http://localhost/nomadxt/api'
+
+
+def upload_file(file_path, name=None):
+    """
+    Upload a file to nomad.
+
+    Arguments:
+        file_path: Path to the file, absolute or relative to call directory.
+        name: Optional name, default is the file_path's basename
+    """
+    if name is None:
+        name = os.path.basename(file_path)
+
+    upload = requests.post('%s/uploads' % api_base, data={name: name}).json()
+
+    upload_cmd = upload['upload_command']
+    upload_cmd = upload_cmd.replace('your_file', file_path)
+
+    subprocess.call(shlex.split(upload_cmd))
+
+    print('File uploaded')
+
+    while True:
+        upload = requests.get('%s/uploads/%s' % (api_base, upload['upload_id'])).json()
+        status = upload['status']
+        calcs_pagination = upload['calcs'].get('pagination')
+        if calcs_pagination is None:
+            total, successes, failures = 0, 0, 0
+        else:
+            total, successes, failures = (
+                calcs_pagination[key] for key in ('total', 'successes', 'failures'))
+
+        print(
+            'status: %s; task: %s; parsing: %d/%d/%d' %
+            (status, upload['current_task'], successes, failures, total))
+
+        if status in ('SUCCESS', 'FAILURE'):
+            break
+
+        time.sleep(5)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) > 3 or len(sys.argv) == 1:
+        print('usage is: <client> filte_to_upload [upload_name]')
+    else:
+        upload_file(sys.argv[1], sys.argv[2] if len(sys.argv) == 3 else None)

nomad/processing/data.py

@@ -271,6 +271,19 @@ class Upload(Proc):
         with lnr(logger, 'deleting upload'):
             super().delete()
 
+    @classmethod
+    def _external_objects_url(cls, url):
+        """ Replaces the given internal object storage url (minio) with an URL that allows
+            external access.
+        """
+        port_with_colon = ''
+        if config.services.objects_port > 0:
+            port_with_colon = ':%d' % config.services.objects_port
+
+        return url.replace(
+            '%s:%s' % (config.minio.host, config.minio.port),
+            '%s%s%s' % (config.services.objects_host, port_with_colon, config.services.objects_base_path))
+
     @classmethod
     def create(cls, **kwargs) -> 'Upload':
         """
@@ -279,7 +292,7 @@ class Upload(Proc):
         The upload will be already saved to the database.
         """
         self = super().create(**kwargs)
-        self.presigned_url = files.get_presigned_upload_url(self.upload_id)
+        self.presigned_url = cls._external_objects_url(files.get_presigned_upload_url(self.upload_id))
         self.upload_command = files.create_curl_upload_cmd(self.presigned_url, 'your_file')
         self._continue_with('uploading')
         return self

requirements.txt

@@ -17,3 +17,4 @@ structlog
 sphinx
 recommonmark
 sphinxcontrib.httpdomain
+drest
\ No newline at end of file

tests/misc.http

-GET http://enc-staging-nomad.esc.rzg.mpg.de/nomadxt/api/uploads  HTTP/1.1
+GET http://localhost/nomadxt/api/uploads  HTTP/1.1
 
 ###
 
-POST http://enc-staging-nomad.esc.rzg.mpg.de/nomadxt/api/uploads  HTTP/1.1
+POST http://localhost/nomadxt/api/uploads  HTTP/1.1
 content-type: application/json
 
 {