Interactive rebase

.gitmodules

 [submodule "docker/webtop/north-guacamole-client"]
 	path = docker/webtop/north-guacamole-client
 	url = https://github.com/nomad-coe/north-guacamole-client.git
+[submodule "docker/sts/OasisSetup/pynxtools-stm"]
+	path = docker/sts/OasisSetup/pynxtools-stm
+	url = https://github.com/FAIRmat-NFDI/pynxtools-stm.git

docker/sts/OasisSetup/PalmasLab/.dockerignore

+.gitignore
+.git
+.pytest_cache
+__pycache__
+test
+.vscode
+.yarn
+.cache
+

docker/sts/OasisSetup/PalmasLab/configs/nginx.conf

+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    ''      close;
+}
+
+server {
+    listen        80;
+    server_name   localhost;
+    proxy_set_header Host $host;
+
+    gzip_min_length     1000;
+    gzip_buffers        4 8k;
+    gzip_http_version   1.0;
+    gzip_disable        "msie6";
+    gzip_vary           on;
+    gzip on;
+    gzip_proxied any;
+    gzip_types
+        text/css
+        text/javascript
+        text/xml
+        text/plain
+        application/javascript
+        application/x-javascript
+        application/json;
+
+    location / {
+        proxy_pass http://app:8000;
+    }
+
+    location ~ /nomad-oasis\/?(gui)?$ {
+        rewrite ^ /nomad-oasis/gui/ permanent;
+    }
+
+    location /nomad-oasis/gui/ {
+        proxy_intercept_errors on;
+        error_page 404 = @redirect_to_index;
+        proxy_pass http://app:8000;
+    }
+
+    location @redirect_to_index {
+        rewrite ^ /nomad-oasis/gui/index.html break;
+        proxy_pass http://app:8000;
+    }
+
+    location ~ \/gui\/(service-worker\.js|meta\.json)$ {
+        add_header Last-Modified $date_gmt;
+        add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
+        if_modified_since off;
+        expires off;
+        etag off;
+        proxy_pass http://app:8000;
+    }
+
+    location ~ /api/v1/uploads(/?$|.*/raw|.*/bundle?$)  {
+        client_max_body_size 35g;
+        proxy_request_buffering off;
+        proxy_pass http://app:8000;
+    }
+
+    location ~ /api/v1/.*/download {
+        proxy_buffering off;
+        proxy_pass http://app:8000;
+    }
+
+    location /nomad-oasis/north/ {
+        client_max_body_size 500m;
+        proxy_pass http://north:9000;
+
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+        # websocket headers
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+        proxy_set_header X-Scheme $scheme;
+
+        proxy_buffering off;
+    }
+}
+

docker/sts/OasisSetup/PalmasLab/configs/nomad.yaml

+services:
+  api_host: 'localhost'
+  api_base_path: '/nomad-oasis'
+  upload_limit: 10000
+
+oasis:
+  is_oasis: true
+  uses_central_user_management: true
+
+north:
+  jupyterhub_crypt_key: '978bfb2e13a8448a253c629d8dd84ff89587f30e635b753153960930cad9d36d'
+
+meta:
+  deployment: 'oasis'
+  deployment_url: 'https://my-oasis.org/api'
+  maintainer_email: 'me@my-oasis.org'
+
+logstash:
+  enabled: false
+
+mongo:
+    db_name: nomad_oasis_v1
+
+elastic:
+    entries_index: nomad_oasis_entries_v1
+    materials_index: nomad_oasis_materials_v1

docker/sts/OasisSetup/PalmasLab/docker-compose.yaml

+version: "3"
+## ++++++++++++++++
+# Note
+# The image `demo-h5web-in-searchresults_sts_palma_lab_l`has been created (using) locally from
+# Nomad-fair gitlab branch demo-h5web-in-searchresults_sts_palma_lab
+## ++++++++++++++++
+services:
+  # broker for celery
+  rabbitmq:
+    restart: unless-stopped
+    image: rabbitmq:3.11.5
+    container_name: nomad_oasis_rabbitmq
+    environment:
+      - RABBITMQ_ERLANG_COOKIE=SWQOKODSQALRPCLNMEQG
+      - RABBITMQ_DEFAULT_USER=rabbitmq
+      - RABBITMQ_DEFAULT_PASS=rabbitmq
+      - RABBITMQ_DEFAULT_VHOST=/
+    volumes:
+      - rabbitmq:/var/lib/rabbitmq
+    healthcheck:
+      test: ["CMD", "rabbitmq-diagnostics", "--silent", "--quiet", "ping"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  # the search engine
+  elastic:
+    restart: unless-stopped
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
+    container_name: nomad_oasis_elastic
+    environment:
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m
+      - discovery.type=single-node
+    volumes:
+      - elastic:/usr/share/elasticsearch/data
+    healthcheck:
+      test:
+        - "CMD"
+        - "curl"
+        - "--fail"
+        - "--silent"
+        - "http://elastic:9200/_cat/health"
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 60s
+
+  # the user data db
+  mongo:
+    restart: unless-stopped
+    image: mongo:5.0.6
+    container_name: nomad_oasis_mongo
+    environment:
+      - MONGO_DATA_DIR=/data/db
+      - MONGO_LOG_DIR=/dev/null
+    volumes:
+      - mongo:/data/db
+      - ./.volumes/mongo:/backup
+    command: mongod --logpath=/dev/null # --quiet
+    healthcheck:
+      test:
+        - "CMD"
+        - "mongo"
+        - "mongo:27017/test"
+        - "--quiet"
+        - "--eval"
+        - "'db.runCommand({ping:1}).ok'"
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  # nomad worker (processing)
+  worker:
+    restart: unless-stopped
+    image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:demo-h5web-in-searchresults_sts_palma_lab_l
+    container_name: nomad_oasis_worker
+    environment:
+      NOMAD_SERVICE: nomad_oasis_worker
+      NOMAD_RABBITMQ_HOST: rabbitmq
+      NOMAD_ELASTIC_HOST: elastic
+      NOMAD_MONGO_HOST: mongo
+      NOMAD_LOGSTASH_HOST: logtransfer
+    depends_on:
+      rabbitmq:
+        condition: service_healthy
+      elastic:
+        condition: service_healthy
+      mongo:
+        condition: service_healthy
+    volumes:
+      - ./configs/nomad.yaml:/app/nomad.yaml
+      - ./.volumes/fs:/app/.volumes/fs
+    command: python -m celery -A nomad.processing worker -l info -Q celery
+
+  # nomad app (api + proxy)
+  app:
+    restart: unless-stopped
+    image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:demo-h5web-in-searchresults_sts_palma_lab_l
+    container_name: nomad_oasis_app
+    environment:
+      NOMAD_SERVICE: nomad_oasis_app
+      NOMAD_SERVICES_API_PORT: 80
+      NOMAD_FS_EXTERNAL_WORKING_DIRECTORY: "$PWD"
+      NOMAD_RABBITMQ_HOST: rabbitmq
+      NOMAD_ELASTIC_HOST: elastic
+      NOMAD_MONGO_HOST: mongo
+      NOMAD_LOGSTASH_HOST: logtransfer
+      NOMAD_NORTH_HUB_HOST: north
+    depends_on:
+      rabbitmq:
+        condition: service_healthy
+      elastic:
+        condition: service_healthy
+      mongo:
+        condition: service_healthy
+      north:
+        condition: service_started
+    volumes:
+      - ./configs/nomad.yaml:/app/nomad.yaml
+      - ./.volumes/fs:/app/.volumes/fs
+    command: ./run.sh
+    healthcheck:
+      test:
+        - "CMD"
+        - "curl"
+        - "--fail"
+        - "--silent"
+        - "http://localhost:8000/-/health"
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  # nomad remote tools hub (JupyterHUB, e.g. for AI Toolkit)
+  north:
+    restart: unless-stopped
+    image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:demo-h5web-in-searchresults_sts_palma_lab_l
+    container_name: nomad_oasis_north
+    environment:
+      NOMAD_SERVICE: nomad_oasis_north
+      NOMAD_NORTH_DOCKER_NETWORK: nomad_oasis_network
+      NOMAD_NORTH_HUB_CONNECT_IP: north
+      NOMAD_NORTH_HUB_IP: "0.0.0.0"
+      NOMAD_NORTH_HUB_HOST: north
+      NOMAD_SERVICES_API_HOST: app
+      NOMAD_FS_EXTERNAL_WORKING_DIRECTORY: "$PWD"
+      NOMAD_RABBITMQ_HOST: rabbitmq
+      NOMAD_ELASTIC_HOST: elastic
+      NOMAD_MONGO_HOST: mongo
+    volumes:
+      - ./configs/nomad.yaml:/app/nomad.yaml
+      - ./.volumes/fs:/app/.volumes/fs
+      - /var/run/docker.sock:/var/run/docker.sock
+    user: '1000:999'
+    command: python -m nomad.cli admin run hub
+    healthcheck:
+      test:
+        - "CMD"
+        - "curl"
+        - "--fail"
+        - "--silent"
+        - "http://localhost:8081/nomad-oasis/north/hub/health"
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  # nomad logtransfer
+  # to enable the logtransfer service run "docker compose --profile with_logtransfer up"
+  logtransfer:
+    restart: unless-stopped
+    image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:demo-h5web-in-searchresults_sts_palma_lab_l
+    container_name: nomad_oasis_logtransfer
+    environment:
+      NOMAD_SERVICE: nomad_oasis_logtransfer
+      NOMAD_ELASTIC_HOST: elastic
+      NOMAD_MONGO_HOST: mongo
+    depends_on:
+      elastic:
+        condition: service_healthy
+      mongo:
+        condition: service_healthy
+    volumes:
+      - ./configs/nomad.yaml:/app/nomad.yaml
+      - ./.volumes/fs:/app/.volumes/fs
+    command: python -m nomad.cli admin run logtransfer
+    profiles: ["with_logtransfer"]
+
+  # nomad proxy (a reverse proxy for nomad)
+  proxy:
+    restart: unless-stopped
+    image: nginx:1.13.9-alpine
+    container_name: nomad_oasis_proxy
+    command: nginx -g 'daemon off;'
+    volumes:
+      - ./configs/nginx.conf:/etc/nginx/conf.d/default.conf
+    depends_on:
+      app:
+        condition: service_healthy
+      worker:
+        condition: service_started # TODO: service_healthy
+      north:
+        condition: service_healthy
+    ports:
+      - 80:80
+
+volumes:
+  mongo:
+    name: "nomad_oasis_mongo"
+  elastic:
+    name: "nomad_oasis_elastic"
+  rabbitmq:
+    name: "nomad_oasis_rabbitmq"
+  keycloak:
+    name: "nomad_oasis_keycloak"
+
+networks:
+  default:
+    name: nomad_oasis_network

docker/sts/OasisSetup/PalmasLab/dockerfiles/Dockerfile

+## ++++++++++++++++
+# Note
+# The image `demo-h5web-in-searchresults_sts_palma_lab_l`has been created locally from
+# Nomad-fair gitlab branch demo-h5web-in-searchresults_sts_palma_lab
+## ++++++++++++++++
+FROM gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:demo-h5web-in-searchresults_sts_palma_lab_l
+
+# root access
+USER root
+RUN pip install --no-cache-dir --upgrade pip
+
+# Install from PyPI
+RUN pip install pynxtools-stm==1.0.4
+
+# Install from source
+# COPY ../pynxtools-stm /app/pynxtools-stm
+# RUN apt-get update && apt-get install -y \
+#     git 
+
+# # RUN pip install --upgrade pip \
+# #     && pip install git+https://github.com/FAIRmat-NFDI/pynxtools-stm.git
+# WORKDIR /app/pynxtools-stm
+
+# RUN python -m build --sdist
+# RUN pip install dist/pynxtools-stm-0.0.0.tar.gz
+WORKDIR /app
+USER nomad

pynxtools-stm @ 6bbd71b8

+Subproject commit 6bbd71b8e39f5ef6b66756cf54995f12b453a186