diff --git a/Dockerfile b/Dockerfile index 1208c0fb68076d82548570065757ad239e3abb63..f53e1e65a8f2e39a21e873b2c964044651c575a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ # The dockerfile is multistages to getaway with building on a larger base image. # using the base image with most requirements already installed -FROM nomad-xt_requirements:latest as requirements +FROM nomadxt_requirements:latest as requirements # we use slim for the final image FROM python:3.6-slim as final diff --git a/gui/nginx.conf b/gui/nginx.conf index 3aaf2fc647dffb3c149327c78cb398f46a53092d..5db1f4d4fbe287176e7f348eee7828c412b31533 100644 --- a/gui/nginx.conf +++ b/gui/nginx.conf @@ -1,12 +1,16 @@ server { listen 80; + location /nomadxt { + root /app/; + } + location /nomadxt/api { proxy_pass http://api:8000; } - location /nomadxt { - root /app/; + location ~ ^/nomadxt/objects/(.*)$ { + resolver 127.0.0.11 ipv6=off; # docker embedded DNS + proxy_pass http://minio:9000/$1; } - } diff --git a/gui/src/api.js b/gui/src/api.js index db54b608bce35bebfac4e10cae1dcf62348b51a7..743d60c5c98dbee415ef0b18cf1d7f6d6c37d898 100644 --- a/gui/src/api.js +++ b/gui/src/api.js @@ -1,4 +1,4 @@ -import { apiBase } from './config' +import { apiBase, objectsBase } from './config' class Upload { constructor(json) { @@ -9,6 +9,7 @@ class Upload { uploadFile(file) { console.assert(this.presigned_url) console.debug(`Upload ${file} to ${this.presigned_url}.`) + const url = this.presigned_url.replace('https://localhost:9000', objectsBase) return fetch(this.presigned_url, { method: 'PUT', headers: { diff --git a/gui/src/components/Navigation.js b/gui/src/components/Navigation.js index 1e2845066fa54128b94ac006c0cd69292eacf69a..3be5de2ab84a04af89adc0382377ed8bd8b6ebf9 100644 --- a/gui/src/components/Navigation.js +++ b/gui/src/components/Navigation.js @@ -20,7 +20,7 @@ import EncIcon from '@material-ui/icons/Assessment'; import { Link, withRouter } from 'react-router-dom'; import { compose } from 'recompose' import { Avatar, MuiThemeProvider } from '@material-ui/core'; -import { genTheme, repoTheme, archiveTheme, encTheme } from '../config'; +import { genTheme, repoTheme, archiveTheme, encTheme, appBase } from '../config'; const drawerWidth = 200; @@ -182,7 +182,7 @@ class Navigation extends React.Component { <Typography variant="title" color="inherit" noWrap className={classes.flex}> {selected(toolbarTitles)} </Typography> - <Avatar src='/me.jpg'/> + <Avatar src={`${appBase}/me.jpg`}/> </Toolbar> </AppBar> </MuiThemeProvider> diff --git a/gui/src/config.js b/gui/src/config.js index e8db1f86d21f8337e8458e54fd865df5c4a53f7e..00fbcf61fd669bb308e72ac8c04414c429d37fa0 100644 --- a/gui/src/config.js +++ b/gui/src/config.js @@ -5,6 +5,7 @@ import secondary from '@material-ui/core/colors/blueGrey'; import { createMuiTheme } from '@material-ui/core'; export const apiBase = '/nomadxt/api' +export const objectsBase = '/nomadxt/objects' export const appBase = '/nomadxt' export const genTheme = createMuiTheme({ diff --git a/infrastructure/nomadxt/.env_local b/infrastructure/nomadxt/.env_local index abf59bf08bc580fb13eacbdaa117c82a600c8ddf..4624512ec15410731cc8029f4372f3a343582f51 100644 --- a/infrastructure/nomadxt/.env_local +++ b/infrastructure/nomadxt/.env_local @@ -6,4 +6,8 @@ MONGO_HOST_PORT=27017 KIBANA_HOST_PORT=5601 API_HOST_PORT=8000 -GUI_HOST_PORT=8005 +GUI_HOST_PORT=80 + +VOLUME_BINDS=../../.volumes +EXTERNAL_HOST=localhost +EXTERNAL_PORT=80 \ No newline at end of file diff --git a/infrastructure/nomadxt/.env_processing b/infrastructure/nomadxt/.env_processing index 8d90dd347e4029d5376d0d29e8d073bba8e64d5f..de5166f1be1b9ac8bf26aef91b448f58bc780838 100644 --- a/infrastructure/nomadxt/.env_processing +++ b/infrastructure/nomadxt/.env_processing @@ -1,4 +1,4 @@ -MINIO_HOST_PORT=10007 +MINIO_HOST_PORT=10000 RABBITMQ_HOST_PORT=10672 REDIS_HOST_PORT=10379 ELASTIC_HOST_PORT=10200 @@ -8,3 +8,6 @@ KIBANA_HOST_PORT=10601 API_HOST_PORT=8083 GUI_HOST_PORT=0.0.0.0:8003 +VOLUME_BINDS=/scratch/nomadxt/volumes +EXTERNAL_HOST=enc-staging-nomad.esc.rzg.mpg.de +EXTERNAL_PORT=80 \ No newline at end of file diff --git a/infrastructure/nomadxt/docker-compose.yml b/infrastructure/nomadxt/docker-compose.yml index f72d1c115be0b14b04f673a481aac658c38a639c..2df0fdf71ba06ad9dc2c623b42ecc07add42ea88 100644 --- a/infrastructure/nomadxt/docker-compose.yml +++ b/infrastructure/nomadxt/docker-compose.yml @@ -24,7 +24,7 @@ services: ports: - ${MINIO_HOST_PORT}:9000 volumes: - - nomadxt_minio:/data + - ${VOLUME_BINDS}/minio:/data - ../config/minio:/root/.minio environment: - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE @@ -103,9 +103,9 @@ services: - rabbitmq - elastic - mongo - - elk + # - elk volumes: - - ../../.volumes/fs:/app/.volumes/fs + - ${VOLUME_BINDS}/fs:/app/.volumes/fs command: python -m celery worker -l info -A nomad.processing # nomad upload handler @@ -125,7 +125,7 @@ services: - rabbitmq - elastic - mongo - - elk + # - elk command: python -m nomad.processing.handlerdaemon # nomad api @@ -140,13 +140,16 @@ services: - NOMAD_ELASTIC_HOST=elastic - NOMAD_MONGO_HOST=mongo - NOMAD_API_BASE_PATH=/nomadxt/api + - NOMAD_OBJECTS_HOST=${EXTERNAL_HOST} + - NOMAD_OBJECTS_PORT=${EXTERNAL_PORT} + - NOMAD_OBJECTS_BASE_PATH=/nomadxt/objects links: - minio - redis - rabbitmq - elastic - mongo - - elk + # - elk ports: - ${API_HOST_PORT}:8000 command: python -m gunicorn.app.wsgiapp -w 4 -b 0.0.0.0:8000 nomad.api:app @@ -156,6 +159,7 @@ services: build: ../../gui/ links: - api + - minio ports: - ${GUI_HOST_PORT}:80 command: nginx -g "daemon off;" diff --git a/nomad/api.py b/nomad/api.py index 98dc184a9a18c2c3264f9943132ece5b4989ff42..bcfaf15218d499a13a5fc6aec199f540e259c90e 100644 --- a/nomad/api.py +++ b/nomad/api.py @@ -27,6 +27,15 @@ if me is None: me.save() +def _external_objects_url(url): + """ Replaces the given internal object storage url (minio) with an URL that allows + external access. """ + port_with_color = '' if config.services.objects_port > 0 else ':%d' % config.services.objects_port + return url.replace( + '%s:%s' % (config.minio.host, config.minio.port), + '%s:%s%s' % (config.services.objects_host, port_with_color, config.services.objects_base_path)) + + class Uploads(Resource): @staticmethod @@ -40,7 +49,7 @@ class Uploads(Resource): data = { 'name': upload.name, 'upload_id': upload.upload_id, - 'presigned_url': upload.presigned_url, + 'presigned_url': _external_objects_url(upload.presigned_url), 'create_time': upload.create_time.isoformat() if upload.create_time is not None else None, 'upload_time': upload.upload_time.isoformat() if upload.upload_time is not None else None, 'proc_time': upload.proc_time.isoformat() if upload.proc_time is not None else None, @@ -139,7 +148,7 @@ def get_calc(upload_hash, calc_hash): archive_id = '%s/%s' % (upload_hash, calc_hash) logger = get_logger(__name__, archive_id=archive_id) try: - url = files.archive_url(archive_id) + url = _external_objects_url(files.archive_url(archive_id)) return redirect(url, 302) except KeyError: abort(404, message='Archive %s does not exist.' % archive_id) diff --git a/nomad/config.py b/nomad/config.py index 8ad0af1d47a141e5dc9bc2176aefc58eb28b9760..034280531d28ca467c3e9a2cc0de8fc36272cd7a 100644 --- a/nomad/config.py +++ b/nomad/config.py @@ -42,7 +42,7 @@ MongoConfig = namedtuple('MongoConfig', ['host', 'users_db']) LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port']) """ Used to configure and enable/disable the ELK based centralized logging. """ -NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_base_path']) +NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_base_path', 'objects_host', 'objects_port', 'objects_base_path']) """ Used to configure nomad services: worker, handler, api """ files = FilesConfig( @@ -90,5 +90,8 @@ logstash = LogstashConfig( tcp_port=int(os.environ.get('NOMAD_LOGSTASH_TCPPORT', '5000')) ) services = NomadServicesConfig( - api_base_path=os.environ.get('NOMAD_API_BASE_PATH', '/nomadxt/api') + api_base_path=os.environ.get('NOMAD_API_BASE_PATH', ''), + objects_host=os.environ.get('NOMAD_OBJECTS_HOST', 'localhost'), + objects_port=int(os.environ.get('NOMAD_OBJECTS_PORT', -1)), + objects_base_path=os.environ.get('NOMAD_OBJECTS_BASE_PATH', '') )