Commit e28cba81 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added simple client for uploading. Fixed external objects urls, some utils for elastic troubles.

parent 3846a275
......@@ -34,7 +34,6 @@ COPY nomad/dependencies.py nomad/dependencies.py
COPY nomad/config.py nomad/config.py
RUN python nomad/dependencies.py
# last stage is used to install the actual code, nomad user, volumes
FROM final
# transfer installed packages from dependency stage
......@@ -48,6 +47,10 @@ COPY --from=dependencies /install /install
RUN apt-get update && apt-get install -y make
COPY . /app
WORKDIR /app
# copy the meta-info, since it files are loaded via relative paths. TODO that should change.
COPY --from=dependencies /install/.dependencies/nomad-meta-info /app/.dependencies/nomad-meta-info
RUN pip install -e .
WORKDIR /app/docs
RUN make html
......
###
# Disable threshold based trigger for turning elastic indices read only due to low disc space
PUT http://localhost:9200/_cluster/settings HTTP/1.1
content-type: application/json
{
"transient" : {
"cluster.routing.allocation.disk.threshold_enabled" : false
}
}
###
# Make calcs index writeable after accedental getting read only
PUT http://localhost:9200/calcs/_settings HTTP/1.1
content-type: application/json
{
"index.blocks.read_only_allow_delete": null
}
###
# Delete the calc index
DELETE http://localhost:9200/calcs HTTP/1.1
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask import Flask, request, redirect
from flask_restful import Resource, Api, abort
from flask_cors import CORS
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Simple client library for the nomad api that allows to bulk upload files via shell command.
"""
import os.path
import subprocess
import shlex
import time
import sys
import requests
api_base = 'http://localhost/nomadxt/api'
def upload_file(file_path, name=None):
"""
Upload a file to nomad.
Arguments:
file_path: Path to the file, absolute or relative to call directory.
name: Optional name, default is the file_path's basename
"""
if name is None:
name = os.path.basename(file_path)
upload = requests.post('%s/uploads' % api_base, data={name: name}).json()
upload_cmd = upload['upload_command']
upload_cmd = upload_cmd.replace('your_file', file_path)
subprocess.call(shlex.split(upload_cmd))
print('File uploaded')
while True:
upload = requests.get('%s/uploads/%s' % (api_base, upload['upload_id'])).json()
status = upload['status']
calcs_pagination = upload['calcs'].get('pagination')
if calcs_pagination is None:
total, successes, failures = 0, 0, 0
else:
total, successes, failures = (
calcs_pagination[key] for key in ('total', 'successes', 'failures'))
print(
'status: %s; task: %s; parsing: %d/%d/%d' %
(status, upload['current_task'], successes, failures, total))
if status in ('SUCCESS', 'FAILURE'):
break
time.sleep(5)
if __name__ == '__main__':
if len(sys.argv) > 3 or len(sys.argv) == 1:
print('usage is: <client> filte_to_upload [upload_name]')
else:
upload_file(sys.argv[1], sys.argv[2] if len(sys.argv) == 3 else None)
......@@ -271,6 +271,19 @@ class Upload(Proc):
with lnr(logger, 'deleting upload'):
super().delete()
@classmethod
def _external_objects_url(cls, url):
""" Replaces the given internal object storage url (minio) with an URL that allows
external access.
"""
port_with_colon = ''
if config.services.objects_port > 0:
port_with_colon = ':%d' % config.services.objects_port
return url.replace(
'%s:%s' % (config.minio.host, config.minio.port),
'%s%s%s' % (config.services.objects_host, port_with_colon, config.services.objects_base_path))
@classmethod
def create(cls, **kwargs) -> 'Upload':
"""
......@@ -279,7 +292,7 @@ class Upload(Proc):
The upload will be already saved to the database.
"""
self = super().create(**kwargs)
self.presigned_url = files.get_presigned_upload_url(self.upload_id)
self.presigned_url = cls._external_objects_url(files.get_presigned_upload_url(self.upload_id))
self.upload_command = files.create_curl_upload_cmd(self.presigned_url, 'your_file')
self._continue_with('uploading')
return self
......
......@@ -17,3 +17,4 @@ structlog
sphinx
recommonmark
sphinxcontrib.httpdomain
drest
\ No newline at end of file
GET http://enc-staging-nomad.esc.rzg.mpg.de/nomadxt/api/uploads HTTP/1.1
GET http://localhost/nomadxt/api/uploads HTTP/1.1
###
POST http://enc-staging-nomad.esc.rzg.mpg.de/nomadxt/api/uploads HTTP/1.1
POST http://localhost/nomadxt/api/uploads HTTP/1.1
content-type: application/json
{
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment