From 12d0322355028e92c74befeb091f1ebcb1b0977a Mon Sep 17 00:00:00 2001
From: Rubel <rubel.mozumder@outlook.com>
Date: Wed, 7 Feb 2024 15:09:43 +0100
Subject: [PATCH] Interactive rebase
---
.gitmodules | 3 +
docker/sts/OasisSetup/PalmasLab/.dockerignore | 9 +
.../OasisSetup/PalmasLab/configs/nginx.conf | 83 +++++++
.../OasisSetup/PalmasLab/configs/nomad.yaml | 26 +++
.../OasisSetup/PalmasLab/docker-compose.yaml | 221 ++++++++++++++++++
.../PalmasLab/dockerfiles/Dockerfile | 27 +++
docker/sts/OasisSetup/pynxtools-stm | 1 +
7 files changed, 370 insertions(+)
create mode 100644 docker/sts/OasisSetup/PalmasLab/.dockerignore
create mode 100644 docker/sts/OasisSetup/PalmasLab/configs/nginx.conf
create mode 100644 docker/sts/OasisSetup/PalmasLab/configs/nomad.yaml
create mode 100644 docker/sts/OasisSetup/PalmasLab/docker-compose.yaml
create mode 100644 docker/sts/OasisSetup/PalmasLab/dockerfiles/Dockerfile
create mode 160000 docker/sts/OasisSetup/pynxtools-stm
diff --git a/.gitmodules b/.gitmodules
index 95ffce7..2d48c99 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,6 @@
[submodule "docker/webtop/north-guacamole-client"]
path = docker/webtop/north-guacamole-client
url = https://github.com/nomad-coe/north-guacamole-client.git
+[submodule "docker/sts/OasisSetup/pynxtools-stm"]
+ path = docker/sts/OasisSetup/pynxtools-stm
+ url = https://github.com/FAIRmat-NFDI/pynxtools-stm.git
diff --git a/docker/sts/OasisSetup/PalmasLab/.dockerignore b/docker/sts/OasisSetup/PalmasLab/.dockerignore
new file mode 100644
index 0000000..379ebb7
--- /dev/null
+++ b/docker/sts/OasisSetup/PalmasLab/.dockerignore
@@ -0,0 +1,9 @@
+.gitignore
+.git
+.pytest_cache
+__pycache__
+test
+.vscode
+.yarn
+.cache
+
diff --git a/docker/sts/OasisSetup/PalmasLab/configs/nginx.conf b/docker/sts/OasisSetup/PalmasLab/configs/nginx.conf
new file mode 100644
index 0000000..74084c9
--- /dev/null
+++ b/docker/sts/OasisSetup/PalmasLab/configs/nginx.conf
@@ -0,0 +1,83 @@
+map $http_upgrade $connection_upgrade {
+ default upgrade;
+ '' close;
+}
+
+server {
+ listen 80;
+ server_name localhost;
+ proxy_set_header Host $host;
+
+ gzip_min_length 1000;
+ gzip_buffers 4 8k;
+ gzip_http_version 1.0;
+ gzip_disable "msie6";
+ gzip_vary on;
+ gzip on;
+ gzip_proxied any;
+ gzip_types
+ text/css
+ text/javascript
+ text/xml
+ text/plain
+ application/javascript
+ application/x-javascript
+ application/json;
+
+ location / {
+ proxy_pass http://app:8000;
+ }
+
+ location ~ /nomad-oasis\/?(gui)?$ {
+ rewrite ^ /nomad-oasis/gui/ permanent;
+ }
+
+ location /nomad-oasis/gui/ {
+ proxy_intercept_errors on;
+ error_page 404 = @redirect_to_index;
+ proxy_pass http://app:8000;
+ }
+
+ location @redirect_to_index {
+ rewrite ^ /nomad-oasis/gui/index.html break;
+ proxy_pass http://app:8000;
+ }
+
+ location ~ \/gui\/(service-worker\.js|meta\.json)$ {
+ add_header Last-Modified $date_gmt;
+ add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
+ if_modified_since off;
+ expires off;
+ etag off;
+ proxy_pass http://app:8000;
+ }
+
+ location ~ /api/v1/uploads(/?$|.*/raw|.*/bundle?$) {
+ client_max_body_size 35g;
+ proxy_request_buffering off;
+ proxy_pass http://app:8000;
+ }
+
+ location ~ /api/v1/.*/download {
+ proxy_buffering off;
+ proxy_pass http://app:8000;
+ }
+
+ location /nomad-oasis/north/ {
+ client_max_body_size 500m;
+ proxy_pass http://north:9000;
+
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+ # websocket headers
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection $connection_upgrade;
+ proxy_set_header X-Scheme $scheme;
+
+ proxy_buffering off;
+ }
+}
+
diff --git a/docker/sts/OasisSetup/PalmasLab/configs/nomad.yaml b/docker/sts/OasisSetup/PalmasLab/configs/nomad.yaml
new file mode 100644
index 0000000..a031140
--- /dev/null
+++ b/docker/sts/OasisSetup/PalmasLab/configs/nomad.yaml
@@ -0,0 +1,26 @@
+services:
+ api_host: 'localhost'
+ api_base_path: '/nomad-oasis'
+ upload_limit: 10000
+
+oasis:
+ is_oasis: true
+ uses_central_user_management: true
+
+north:
+ jupyterhub_crypt_key: '978bfb2e13a8448a253c629d8dd84ff89587f30e635b753153960930cad9d36d'
+
+meta:
+ deployment: 'oasis'
+ deployment_url: 'https://my-oasis.org/api'
+ maintainer_email: 'me@my-oasis.org'
+
+logstash:
+ enabled: false
+
+mongo:
+ db_name: nomad_oasis_v1
+
+elastic:
+ entries_index: nomad_oasis_entries_v1
+ materials_index: nomad_oasis_materials_v1
diff --git a/docker/sts/OasisSetup/PalmasLab/docker-compose.yaml b/docker/sts/OasisSetup/PalmasLab/docker-compose.yaml
new file mode 100644
index 0000000..6c3a772
--- /dev/null
+++ b/docker/sts/OasisSetup/PalmasLab/docker-compose.yaml
@@ -0,0 +1,221 @@
+version: "3"
+## ++++++++++++++++
+# Note
+# The image `demo-h5web-in-searchresults_sts_palma_lab_l`has been created (using) locally from
+# Nomad-fair gitlab branch demo-h5web-in-searchresults_sts_palma_lab
+## ++++++++++++++++
+services:
+ # broker for celery
+ rabbitmq:
+ restart: unless-stopped
+ image: rabbitmq:3.11.5
+ container_name: nomad_oasis_rabbitmq
+ environment:
+ - RABBITMQ_ERLANG_COOKIE=SWQOKODSQALRPCLNMEQG
+ - RABBITMQ_DEFAULT_USER=rabbitmq
+ - RABBITMQ_DEFAULT_PASS=rabbitmq
+ - RABBITMQ_DEFAULT_VHOST=/
+ volumes:
+ - rabbitmq:/var/lib/rabbitmq
+ healthcheck:
+ test: ["CMD", "rabbitmq-diagnostics", "--silent", "--quiet", "ping"]
+ interval: 10s
+ timeout: 10s
+ retries: 30
+ start_period: 10s
+
+ # the search engine
+ elastic:
+ restart: unless-stopped
+ image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
+ container_name: nomad_oasis_elastic
+ environment:
+ - ES_JAVA_OPTS=-Xms512m -Xmx512m
+ - discovery.type=single-node
+ volumes:
+ - elastic:/usr/share/elasticsearch/data
+ healthcheck:
+ test:
+ - "CMD"
+ - "curl"
+ - "--fail"
+ - "--silent"
+ - "http://elastic:9200/_cat/health"
+ interval: 10s
+ timeout: 10s
+ retries: 30
+ start_period: 60s
+
+ # the user data db
+ mongo:
+ restart: unless-stopped
+ image: mongo:5.0.6
+ container_name: nomad_oasis_mongo
+ environment:
+ - MONGO_DATA_DIR=/data/db
+ - MONGO_LOG_DIR=/dev/null
+ volumes:
+ - mongo:/data/db
+ - ./.volumes/mongo:/backup
+ command: mongod --logpath=/dev/null # --quiet
+ healthcheck:
+ test:
+ - "CMD"
+ - "mongo"
+ - "mongo:27017/test"
+ - "--quiet"
+ - "--eval"
+ - "'db.runCommand({ping:1}).ok'"
+ interval: 10s
+ timeout: 10s
+ retries: 30
+ start_period: 10s
+
+ # nomad worker (processing)
+ worker:
+ restart: unless-stopped
+ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:demo-h5web-in-searchresults_sts_palma_lab_l
+ container_name: nomad_oasis_worker
+ environment:
+ NOMAD_SERVICE: nomad_oasis_worker
+ NOMAD_RABBITMQ_HOST: rabbitmq
+ NOMAD_ELASTIC_HOST: elastic
+ NOMAD_MONGO_HOST: mongo
+ NOMAD_LOGSTASH_HOST: logtransfer
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+ elastic:
+ condition: service_healthy
+ mongo:
+ condition: service_healthy
+ volumes:
+ - ./configs/nomad.yaml:/app/nomad.yaml
+ - ./.volumes/fs:/app/.volumes/fs
+ command: python -m celery -A nomad.processing worker -l info -Q celery
+
+ # nomad app (api + proxy)
+ app:
+ restart: unless-stopped
+ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:demo-h5web-in-searchresults_sts_palma_lab_l
+ container_name: nomad_oasis_app
+ environment:
+ NOMAD_SERVICE: nomad_oasis_app
+ NOMAD_SERVICES_API_PORT: 80
+ NOMAD_FS_EXTERNAL_WORKING_DIRECTORY: "$PWD"
+ NOMAD_RABBITMQ_HOST: rabbitmq
+ NOMAD_ELASTIC_HOST: elastic
+ NOMAD_MONGO_HOST: mongo
+ NOMAD_LOGSTASH_HOST: logtransfer
+ NOMAD_NORTH_HUB_HOST: north
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+ elastic:
+ condition: service_healthy
+ mongo:
+ condition: service_healthy
+ north:
+ condition: service_started
+ volumes:
+ - ./configs/nomad.yaml:/app/nomad.yaml
+ - ./.volumes/fs:/app/.volumes/fs
+ command: ./run.sh
+ healthcheck:
+ test:
+ - "CMD"
+ - "curl"
+ - "--fail"
+ - "--silent"
+ - "http://localhost:8000/-/health"
+ interval: 10s
+ timeout: 10s
+ retries: 30
+ start_period: 10s
+
+ # nomad remote tools hub (JupyterHUB, e.g. for AI Toolkit)
+ north:
+ restart: unless-stopped
+ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:demo-h5web-in-searchresults_sts_palma_lab_l
+ container_name: nomad_oasis_north
+ environment:
+ NOMAD_SERVICE: nomad_oasis_north
+ NOMAD_NORTH_DOCKER_NETWORK: nomad_oasis_network
+ NOMAD_NORTH_HUB_CONNECT_IP: north
+ NOMAD_NORTH_HUB_IP: "0.0.0.0"
+ NOMAD_NORTH_HUB_HOST: north
+ NOMAD_SERVICES_API_HOST: app
+ NOMAD_FS_EXTERNAL_WORKING_DIRECTORY: "$PWD"
+ NOMAD_RABBITMQ_HOST: rabbitmq
+ NOMAD_ELASTIC_HOST: elastic
+ NOMAD_MONGO_HOST: mongo
+ volumes:
+ - ./configs/nomad.yaml:/app/nomad.yaml
+ - ./.volumes/fs:/app/.volumes/fs
+ - /var/run/docker.sock:/var/run/docker.sock
+ user: '1000:999'
+ command: python -m nomad.cli admin run hub
+ healthcheck:
+ test:
+ - "CMD"
+ - "curl"
+ - "--fail"
+ - "--silent"
+ - "http://localhost:8081/nomad-oasis/north/hub/health"
+ interval: 10s
+ timeout: 10s
+ retries: 30
+ start_period: 10s
+
+ # nomad logtransfer
+ # to enable the logtransfer service run "docker compose --profile with_logtransfer up"
+ logtransfer:
+ restart: unless-stopped
+ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:demo-h5web-in-searchresults_sts_palma_lab_l
+ container_name: nomad_oasis_logtransfer
+ environment:
+ NOMAD_SERVICE: nomad_oasis_logtransfer
+ NOMAD_ELASTIC_HOST: elastic
+ NOMAD_MONGO_HOST: mongo
+ depends_on:
+ elastic:
+ condition: service_healthy
+ mongo:
+ condition: service_healthy
+ volumes:
+ - ./configs/nomad.yaml:/app/nomad.yaml
+ - ./.volumes/fs:/app/.volumes/fs
+ command: python -m nomad.cli admin run logtransfer
+ profiles: ["with_logtransfer"]
+
+ # nomad proxy (a reverse proxy for nomad)
+ proxy:
+ restart: unless-stopped
+ image: nginx:1.13.9-alpine
+ container_name: nomad_oasis_proxy
+ command: nginx -g 'daemon off;'
+ volumes:
+ - ./configs/nginx.conf:/etc/nginx/conf.d/default.conf
+ depends_on:
+ app:
+ condition: service_healthy
+ worker:
+ condition: service_started # TODO: service_healthy
+ north:
+ condition: service_healthy
+ ports:
+ - 80:80
+
+volumes:
+ mongo:
+ name: "nomad_oasis_mongo"
+ elastic:
+ name: "nomad_oasis_elastic"
+ rabbitmq:
+ name: "nomad_oasis_rabbitmq"
+ keycloak:
+ name: "nomad_oasis_keycloak"
+
+networks:
+ default:
+ name: nomad_oasis_network
diff --git a/docker/sts/OasisSetup/PalmasLab/dockerfiles/Dockerfile b/docker/sts/OasisSetup/PalmasLab/dockerfiles/Dockerfile
new file mode 100644
index 0000000..7663c16
--- /dev/null
+++ b/docker/sts/OasisSetup/PalmasLab/dockerfiles/Dockerfile
@@ -0,0 +1,27 @@
+## ++++++++++++++++
+# Note
+# The image `demo-h5web-in-searchresults_sts_palma_lab_l`has been created locally from
+# Nomad-fair gitlab branch demo-h5web-in-searchresults_sts_palma_lab
+## ++++++++++++++++
+FROM gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:demo-h5web-in-searchresults_sts_palma_lab_l
+
+# root access
+USER root
+RUN pip install --no-cache-dir --upgrade pip
+
+# Install from PyPI
+RUN pip install pynxtools-stm==1.0.4
+
+# Install from source
+# COPY ../pynxtools-stm /app/pynxtools-stm
+# RUN apt-get update && apt-get install -y \
+# git
+
+# # RUN pip install --upgrade pip \
+# # && pip install git+https://github.com/FAIRmat-NFDI/pynxtools-stm.git
+# WORKDIR /app/pynxtools-stm
+
+# RUN python -m build --sdist
+# RUN pip install dist/pynxtools-stm-0.0.0.tar.gz
+WORKDIR /app
+USER nomad
diff --git a/docker/sts/OasisSetup/pynxtools-stm b/docker/sts/OasisSetup/pynxtools-stm
new file mode 160000
index 0000000..6bbd71b
--- /dev/null
+++ b/docker/sts/OasisSetup/pynxtools-stm
@@ -0,0 +1 @@
+Subproject commit 6bbd71b8e39f5ef6b66756cf54995f12b453a186
--
GitLab