Added gui code to pypi package. Added documentation for bare-metal oasis install.

.dockerignore

@@ -3,7 +3,7 @@
 **/.mypy_cache
 **/__pycache__
 **/*.pyc
-**/NOMAD.egg-info
+**/*.egg-info
 .*env/
 .pyenv*/
 .pytest_cache
@@ -14,6 +14,8 @@
 **/.ipynb_checkpoints
 **/.volumes
 
+nomad_lab.egg-info/
+
 data/
 local/
 target/
@@ -30,6 +32,7 @@ docs/*.graffle
 
 nomad/normalizing/data/*.db
 nomad/normalizing/data/*.msg
+nomad/app/static
 nomad.yaml
 
 gui/node_modules/

Dockerfile

@@ -69,8 +69,6 @@ WORKDIR /install
 COPY . /install
 RUN python setup.py compile
 RUN pip install .[all]
-RUN python setup.py sdist
-RUN cp dist/nomad-lab-*.tar.gz dist/nomad-lab.tar.gz
 RUN python -m nomad.cli dev metainfo > gui/src/metainfo.json
 RUN python -m nomad.cli dev search-quantities > gui/src/searchQuantities.json
 RUN python -m nomad.cli dev toolkit-metadata > gui/src/toolkitMetadata.json
@@ -115,24 +113,21 @@ COPY . /app
 WORKDIR /app
 # transfer installed packages from dependency stage
 COPY --from=build /usr/local/lib/python3.7/site-packages /usr/local/lib/python3.7/site-packages
-RUN echo "copy 1"
 # copy the documentation, its files will be served by the API
 COPY --from=build /install/docs/.build /app/docs/.build
-RUN echo "copy 2"
-# copy the source distribution, its files will be served by the API
-COPY --from=build /install/dist /app/dist
-RUN echo "copy 3"
 # copy the nomad command
 COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
-RUN echo "copy 4"
 # copy the gui
 RUN mkdir -p /app/gui
-COPY --from=gui_build /app/build /app/gui/build
-RUN echo "copy 5"
+COPY --from=gui_build /app/build /app/nomad/app/static/gui
 # copy the encyclopedia gui production code
-COPY --from=gui_build /encyclopedia /app/dependencies/encyclopedia-gui/client
-RUN rm -rf /app/dependencies/encyclopedia-gui/client/src
-RUN echo "copy 6"
+COPY --from=gui_build /encyclopedia /app/nomad/app/static/encyclopedia
+# remove the developer config on the gui, will be generated by run.sh from nomad.yaml
+RUN rm -f /app/nomad/app/static/gui/env.js
+RUN rm -f /app/nomad/app/static/encyclopedia/conf.js
+# build the python package dist
+RUN python setup.py sdist
+RUN cp dist/nomad-lab-*.tar.gz dist/nomad-lab.tar.gz
 
 RUN mkdir -p /app/.volumes/fs
 RUN useradd -ms /bin/bash nomad

MANIFEST.in

@@ -6,3 +6,4 @@ include LICENSE.txt
 include requirements.txt
 include auto_complete_install.sh
 include setup.json
+recursive-include nomad/app/static *.css *.ico *.html *.json *.js *.map *.txt *.svg *.png
\ No newline at end of file

example @ c378ce3c

-Subproject commit b512a2ae5acc3e35887c291a59d03f81169e2af3
+Subproject commit c378ce3c667fa0b6bd93ecdfd5699b7b9e165fa5

vasp @ 56eb84ff

-Subproject commit 10973cb74a154c5896633cb50f6e576fddd72852
+Subproject commit 56eb84fffa3ae25a4b8dc10aa56da85a4fe6584d

nomad/app/.gitignore

+static
\ No newline at end of file

nomad/app/encyclopedia.py

@@ -19,6 +19,5 @@
 from flask import Blueprint
 import os.path
 
-gui_folder = os.path.abspath(os.path.join(
-    os.path.dirname(__file__), '../../dependencies/encyclopedia-gui/client'))
+gui_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), 'static/encyclopedia'))
 blueprint = Blueprint('encyclopedia', __name__, static_url_path='/', static_folder=gui_folder)

nomad/app/gui.py

@@ -19,8 +19,10 @@
 from flask import Blueprint, request
 import os.path
 
-gui_folder = os.path.abspath(os.path.join(
-    os.path.dirname(__file__), '../../gui/build'))
+gui_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), 'static/gui'))
+configuired_gui_folder = os.path.join(gui_folder, '../.gui_configured')
+if os.path.exists(configuired_gui_folder):
+    gui_folder = configuired_gui_folder
 blueprint = Blueprint('gui', __name__, static_url_path='/', static_folder=gui_folder)
 
 

nomad/cli/admin/admin.py

@@ -612,8 +612,10 @@ def restore(path_to_dump):
 
 
 @ops.command(help=('Generate an nginx.conf to serve the GUI and proxy pass to API container.'))
-@click.option('--prefix', type=str, default='/example_nomad', help='Url path prefix. Default is /example_nomd, can be empty str.')
-def nginx_conf(prefix):
+@click.option('--prefix', type=str, default=config.services.api_base_path, help='Alter the url path prefix.')
+@click.option('--host', type=str, default=config.services.api_host, help='Alter the NOMAD app host.')
+@click.option('--port', type=str, default=config.services.api_port, help='Alter the NOMAD port host.')
+def nginx_conf(prefix, host, port):
     prefix = prefix.rstrip('/')
     prefix = '/%s' % prefix.lstrip('/')
 
@@ -624,22 +626,22 @@ server {{
     proxy_set_header Host $host;
 
     location / {{
-        proxy_pass http://app:8000;
+        proxy_pass http://{1}:{2};
     }}
 
-    location ~ {1}\\/?(gui)?$ {{
-        rewrite ^ {1}/gui/ permanent;
+    location ~ {0}\\/?(gui)?$ {{
+        rewrite ^ {0}/gui/ permanent;
     }}
 
-    location {1}/gui/ {{
+    location {0}/gui/ {{
         proxy_intercept_errors on;
         error_page 404 = @redirect_to_index;
-        proxy_pass http://app:8000;
+        proxy_pass http://{1}:{2};
     }}
 
     location @redirect_to_index {{
-        rewrite ^ {1}/gui/index.html break;
-        proxy_pass http://app:8000;
+        rewrite ^ {0}/gui/index.html break;
+        proxy_pass http://{1}:{2};
     }}
 
     location ~ \\/gui\\/(service-worker\\.js|meta\\.json)$ {{
@@ -648,69 +650,37 @@ server {{
         if_modified_since off;
         expires off;
         etag off;
-        proxy_pass http://app:8000;
+        proxy_pass http://{1}:{2};
     }}
 
     location ~ \\/api\\/uploads\\/?$ {{
         client_max_body_size 35g;
         proxy_request_buffering off;
-        proxy_pass http://app:8000;
+        proxy_pass http://{1}:{2};
     }}
 
     location ~ \\/api\\/(raw|archive) {{
         proxy_buffering off;
-        proxy_pass http://app:8000;
+        proxy_pass http://{1}:{2};
     }}
 
     location ~ \\/api\\/mirror {{
         proxy_buffering off;
         proxy_read_timeout 600;
-        proxy_pass http://app:8000;
+        proxy_pass http://{1}:{2};
     }}
-}}'''.format(prefix))
 
+    location ~ \\/encyclopedia\\/ {{
+        proxy_intercept_errors on;
+        error_page 404 = @redirect_to_encyclopedia_index;
+        proxy_pass http://{1}:{2};
+    }}
 
-@ops.command(help=('Generate a proxy pass config for apache2 reverse proxy servers.'))
-@click.option('--prefix', type=str, default='app', help='The path prefix under which everything is proxy passed.')
-@click.option('--host', type=str, default='130.183.207.104', help='The host to proxy to.')
-@click.option('--port', type=str, default='30001', help='The port to proxy to.')
-def apache_conf(prefix, host, port):
-    print('''\
-ProxyPass "/{0}" "http://{1}:{2}/{0}"
-ProxyPassReverse "/{0}" "http://{1}:{2}/{0}"
-<Proxy http://{1}:{2}/{0}>
-    ProxyPreserveHost On
-    <IfModule !mod_access_compat.c>
-         Require all granted
-     </IfModule>
-     <IfModule mod_access_compat.c>
-         Order allow,deny
-         Allow from all
-     </IfModule>
-</Proxy>
-
-RequestHeader set "X-Forwarded-Proto" expr=%{{REQUEST_SCHEME}}
-RequestHeader set "X-Forwarded-SSL" expr=%{{HTTPS}}
-
-ProxyPass /fairdi/keycloak http://{1}:8002/fairdi/keycloak
-ProxyPassReverse /fairdi/keycloak http://{1}:8002/fairdi/keycloak
-<Proxy http://{1}:8002/app>
-     ProxyPreserveHost On
-     <IfModule !mod_access_compat.c>
-         Require all granted
-     </IfModule>
-     <IfModule mod_access_compat.c>
-         Order allow,deny
-         Allow from all
-     </IfModule>
-</Proxy>
-
-RewriteEngine on
-RewriteCond %{QUERY_STRING} ^pid=([^&]+)$
-RewriteRule ^/NomadRepository-1.1/views/calculation.zul$ /{0}/gui/entry/pid/%1? [R=301]
-
-AllowEncodedSlashes On
-'''.format(prefix, host, port))  # type: ignore
+    location @redirect_to_encyclopedia_index {{
+        rewrite ^ {0}/encyclopedia/index.html break;
+        proxy_pass http://{1}:{2};
+    }}
+}}'''.format(prefix, host, port))
 
 
 @ops.command(help='Updates the AFLOW prototype information using the latest online version and writes the results to a python module in the given FILEPATH.')
@@ -746,3 +716,76 @@ def update(input_dir, out, verbose):
 def ingest(input_path, batch_size, verbose):
     from nomad.cli.admin import similarity
     similarity.ingest(input_path, batch_size, verbose)
+
+
+@ops.command(help='Configures the GUIs based on NOMAD config.')
+def gui_config():
+    import os
+    import os.path
+    from nomad import config
+    import glob
+    import shutil
+
+    gui_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../app/static/gui'))
+    run_gui_folder = os.path.join(gui_folder, '../.gui_configured')
+
+    # copy
+    shutil.rmtree(run_gui_folder, ignore_errors=True)
+    shutil.copytree(gui_folder, run_gui_folder)
+
+    # setup the env
+    env_js_file = os.path.join(run_gui_folder, 'env.js')
+    if not os.path.exists(env_js_file):
+        with open(env_js_file, 'wt') as f:
+            f.write(('''
+window.nomadEnv = {
+    'appBase': '%s',
+    'keycloakBase': 'https://nomad-lab.eu/fairdi/keycloak/auth/',
+    'keycloakRealm': '%s',
+    'keycloakClientId': '%s',
+    'debug': false,
+    'matomoEnabled': false,
+    'encyclopediaEnabled': true,
+    'oasis': %s
+};''' % (
+                config.services.api_base_path,
+                config.keycloak.realm_name,
+                config.keycloak.client_id,
+                'true' if config.keycloak.oasis else 'false'
+            )))
+
+    # replace base path in all GUI files
+    source_file_globs = [
+        '**/*.json',
+        '**/*.html',
+        '**/*.js',
+        '**/*.js.map',
+        '**/*.css']
+    for source_file_glob in source_file_globs:
+        source_files = glob.glob(os.path.join(run_gui_folder, source_file_glob), recursive=True)
+        for source_file in source_files:
+            with open(source_file, 'rt') as f:
+                file_data = f.read()
+            file_data = file_data.replace('/fairdi/nomad/latest', config.services.api_base_path)
+            with open(source_file, 'wt') as f:
+                f.write(file_data)
+
+    gui_folder = os.path.abspath(os.path.join(
+        os.path.dirname(__file__), '../../app/static/encyclopedia'))
+
+    # setup the env
+    conf_js_file = os.path.join(gui_folder, 'conf.js')
+    if not os.path.exists(conf_js_file):
+        with open(conf_js_file, 'wt') as f:
+            f.write(('''
+window.nomadEnv = {
+    apiRoot: "%s/api/encyclopedia/",
+    keycloakBase: "%s",
+    keycloakRealm: "%s",
+    keycloakClientId: "%s"
+};''' % (
+                config.services.api_base_path,
+                config.keycloak.server_url,
+                config.keycloak.realm_name,
+                config.keycloak.client_id
+            )))

ops/docker-compose/nomad-oasis/README.md

 # Operating an OASIS
 
-The following describes the simplest way to run your own NOMAD.
-
-## What is an OASIS
-
 Originally, NOMAD Central Repository is a service run at Max-Planck's computing facility in Garching, Germany.
 However, the NOMAD software is Open-Source, and everybody can run it. Any service that
 uses NOMAD software independently is called a *NOMAD OASIS*.
@@ -11,8 +7,32 @@ uses NOMAD software independently is called a *NOMAD OASIS*.
 While several use cases require different setups, this documentation
 describes the simplest setup of a NOMAD OASIS. It would allow a group to use NOMAD for
 local research data management, while using NOMAD's central user-management and its users.
+There are several environment in which you can run a NOMAD OASIS: base-metal linux,
+docker with docker-compose, and in a kubernetes cluster. We recommend using docker/docker-compose.
+
+## Before you start
+
+We need to register some information about your OASIS in the central user management:
+- The hostname for the machine you run NOMAD on. This is important for redirects between
+your OASIS and the central NOMAD user-management and to allow your users to upload files (via GUI or API).
+Your machine needs to be accessible under this hostname from the public internet. The host
+name needs to be registered with the central NOMAD in order to configure the central user-
+management correctly.
+- Your NOMAD account that should act as an admin account for your OASIS. This account must be declared
+to the central NOMAD as an OASIS admin in order to give it the necessary rights in the central user-
+management.
+- You will need to know your NOMAD user-id. This information has to be provided by us.
+
+Please [write us](mailto:support@nomad-lab.eu) to register your NOMAD account as an OASIS
+admin and to register your hostname. Please replace the indicated configuration items with
+the right information.
+
+
+In principle, you can also run your own user management. This is not yet documented.
 
-## Pre-requisites
+## docker and docker compose
+
+### Pre-requisites
 
 NOMAD software is distributed as a set of docker containers. Further, other services
 that can be run with docker are required. Further, we use docker-compose to setup
@@ -25,30 +45,15 @@ database, an **elasticsearch**, a **rabbitmq** distributed task queue, the NOMAD
 NOMAD **worker**, and NOMAD **gui**. Refer to this [introduction](/app/docs/introduction.html#architecture)
 to learn what each service does and why it is necessary.
 
-There is also some information you need to configure your NOMAD OASIS:
-- The hostname for the machine you run NOMAD on. This is important for redirects between
-your OASIS and the central NOMAD user-management and to allow your users to upload files (via GUI or API).
-Your machine needs to be accessible under this hostname from the public internet. The host
-name needs to be registered with the central NOMAD in order to configure the central user-
-management correctly.
-- A NOMAD account that acts as an admin account for your OASIS. This account must be declared
-to the central NOMAD as an OASIS admin in order to give it the necessary rights in the central user-
-management.
-
-## Configuration
+### Configuration overview
 
 All docker container are configured via docker-compose an the respective `docker-compose.yaml` file.
 Further, we will need to mount some configuration files to configure the NOMAD services within
 their respective containers.
 
-Please [write us](mailto:support@nomad-lab.eu) to register your NOMAD account as an OASIS
-admin and to register your hostname. Please replace the indicated configuration items with
-the right information.
-
 There are three files to configure:
 - `docker-compose.yaml`
 - `nomad.yaml`
-- `env.js`
 - `nginx.conf`
 
 In this example, we have all files in the same directory (the directory we also work from).
@@ -134,9 +139,8 @@ services:
             - mongo
         volumes:
             - ./nomad.yaml:/app/nomad.yaml
-            - ./env.js:/app/gui/build/env.js
             - nomad_oasis_files:/app/.volumes/fs
-        command: ["./run.sh", "/nomad-oasis"]
+        command: gunicorn -b 0.0.0.0 nomad.app:app
 
     # nomad gui (a reverse proxy for nomad)
     gui:
@@ -202,24 +206,6 @@ You need to change:
 A few things to notice:
 - Be secretive about your admin credentials; make sure this file is not publicly readable.
 
-### env.js
-
-The GUI also has a config file, called `env.js` with a similar function than `nomad.yaml`.
-
-```js
-window.nomadEnv = {
-  'appBase': '/nomad-oasis/',
-  'keycloakBase': 'https://nomad-lab.eu/fairdi/keycloak/auth/',
-  'keycloakRealm': 'fairdi_nomad_prod',
-  'keycloakClientId': 'nomad_public',
-  'debug': false,
-  'oasis': true
-};
-```
-
-You need to change:
-- `appBase` defines the base path again. It needs to be changed, if you use a different base path.
-
 ### nginx.conf
 
 The GUI container serves as a proxy that forwards request to the app container. The
@@ -293,7 +279,7 @@ Gunicorn is the WSGI-server that runs the nomad app. Consult the
 [gunicorn documentation](https://docs.gunicorn.org/en/stable/configure.html) for
 configuration options.
 
-## Starting and stopping
+### Starting and stopping NOMAD
 
 If you prepared the above files, simply use the usual `docker-compose` commands to start everything.
 
@@ -349,8 +335,116 @@ If you want to report problems with your OASIS. Please provide the logs for
 - nomad_oasis_worker
 - nomad_oasis_gui
 
+## base linux (without docker)
+
+### Pre-requisites
+
+We will run NOMAD from the *nomad-lab* Python package. This package contains all the necessary
+code for running the processing, api, and gui. But, it is missing the necessary databases. You might be 
+able to run NOMAD in user space.
+
+You will need:
+
+- preferably a linux computer, which Python 3.7, preferable a Python virtual environment
+- elasticsearch 6.x, running without users and authentication, preferable on the default settings
+- mongodb 4.x, running without users and authentication, preferable on the default settings
+- rabbitmq 3.x, running without users and authentication, preferable on the default settings
+- nginx
+- an empty directory to work in
+
+### Install the NOMAD Python package
+
+```sh
+virtualenv -p `which python3` nomadpyenv
+source nomadpyenv/bin/activate
+```
+
+```sh
+pip install nomad-lab[all]
+```
+
+```sh
+curl "https://nomad-lab.eu/prod/rae/beta/dist/nomad-lab.tar.gz" -o nomad-lab.tar.gz
+pip install nomad-lab.tar.gz[all]
+```
+
+### Configure NOMAD - nomad.yaml
+
+```yaml
+client:
+  url: 'http://<your-host>/nomad-oasis/api'
+
+services:
+  api_base_path: '/nomad-oasis'
+  admin_user_id: '<your admin user id>'
+
+keycloak:
+  realm_name: fairdi_nomad_prod
+  username: '<your admin username>'
+  password: '<your admin user password>'
+  oasis: true
+
+mongo:
+    db_name: nomad_v0_8
+
+elastic:
+    index_name: nomad_v0_8
+```
+
+You need to change:
+- Replace `your-host` and admin credentials respectively.
+- `api_base_path` defines the path under with the app is run. It needs to be changed, if you use a different base path.
+
+A few things to notice:
+- Be secretive about your admin credentials; make sure this file is not publicly readable.
+
+### Configure NOMAD - nginx
+
+You can generate a suitable `nginx.conf` with the `nomad` command line command that
+comes with the *nomad-lab* Python package. If you server other content but NOMAD with
+your nginx, you need to incorporate the config accordingly.
+
+If you have a standard installation of nginx, this might work. Adapt to your set-up:
+```sh
+nomad admin ops nginx-conf > /etc/nginx/conf.d/default.conf
+nginx -t
+nginx -s reload
+```
+
+If you want to run nginx in docker, this might work. Adapt to your set-up:
+```sh
+nomad admin ops nginx-conf --host host.docker.internal > nginx.conf
+docker run --rm -v `pwd`/nginx.conf:/etc/nginx/conf.d/default.conf -p 80:80 nginx:stable nginx -g 'daemon off;' &
+```
+
+### Running NOMAD
+
+Before you start, we need to transfer your `nomad.yaml` config values to the GUI's
+javascript. You need to repeat this, if you change your `nomad.yaml`. You can do this by running:
+```
+nomad admin ops gui-config
+```
+
+To run NOMAD, you must run two services. One is the NOMAD app, it serves the API and GUI:
+```
+gunicorn "${params[@]}" -b 0.0.0.0:8000 nomad.app:app
+```
+
+And the NOMAD worker, that runs the NOMAD processing.
+```
+celery worker -l info -A nomad.processing -Q celery,calcs,uploads
+```
+
+This should give you a working OASIS at `http://<your-host>/<your-path-prefix>`.
+
+## kubernetes
+
+*This is not yet documented.*
+
 ## Migrating from an older version (0.7.x to 0.8.x)
 
+*This documentation is outdated and will be removed in future releases.*
+
 Between versions 0.7.x and 0.8.x we needed to change how archive and metadata data is stored
 internally in files and databases. This means you cannot simply start a new version of
 NOMAD on top of the old data. But there is a strategy to adapt the data.
@@ -361,7 +455,7 @@ docker-compose stop
 docker-compose rm -f
 ```
 
-Update you config files (`docker-compose.yaml`, `nomad.yaml`, `env.js`, `nginx.conf`) according
+Update you config files (`docker-compose.yaml`, `nomad.yaml`, `nginx.conf`) according
 to the latest documentation (see above). Especially make sure to select a new name for
 databases and search index in your `nomad.yaml` (e.g. `nomad_v0_8`). This will
 allow us to create new data while retaining the old, i.e. to copy the old data over.

ops/docker-compose/nomad-oasis/docker-compose.yml

@@ -72,9 +72,8 @@ services:
             - mongo
         volumes:
             - ./nomad.yaml:/app/nomad.yaml
-            - ./env.js:/app/gui/build/env.js
             - nomad_oasis_files:/app/.volumes/fs
-        command: ["./run.sh", "/nomad-oasis"]
+        command: ./run.sh
 
     # nomad gui (a reverse proxy for nomad)
     gui:

ops/docker-compose/nomad-oasis/env.js

-window.nomadEnv = {
-  'appBase': '/nomad-oasis/',
-  'keycloakBase': 'https://nomad-lab.eu/fairdi/keycloak/auth/',
-  'keycloakRealm': 'fairdi_nomad_prod',
-  'keycloakClientId': 'nomad_public',
-  'debug': false,
-  'matomoEnabled': false,
-  'encyclopediaEnabled': false,
-  'oasis': true
-};
\ No newline at end of file

ops/docker-compose/nomad-oasis/nginx.conf

@@ -47,4 +47,15 @@ server {
         proxy_read_timeout 600;
         proxy_pass http://app:8000;
     }
+
+    location ~ \/encyclopedia\/ {
+        proxy_intercept_errors on;
+        error_page 404 = @redirect_to_encyclopedia_index;
+        proxy_pass http://app:8000;
+    }
+
+    location @redirect_to_encyclopedia_index {
+        rewrite ^ /nomad-oasis/encyclopedia/index.html break;
+        proxy_pass http://app:8000;
+    }
 }
\ No newline at end of file

ops/helm/nomad/templates/api-deployment.yaml

@@ -168,11 +168,11 @@ spec:
           name: staging-volume
         - mountPath: /nomad
           name: nomad-volume
-        - mountPath: /app/gui/build/env.js
+        - mountPath: /app/nomad/static/gui/env.js
           readOnly: true
           subPath: env.js
           name: gui-env-js
-        - mountPath: /app/dependencies/encyclopedia-gui/client/conf.js
+        - mountPath: /app/nomad/static/encyclopedia/conf.js
           readOnly: true
           subPath: conf.js
           name: enc-conf-js
@@ -223,7 +223,7 @@ spec:
               name: {{ .Values.datacite.secret }}
               key: user
         {{ end }}
-        command: ["./run.sh",  "{{ .Values.proxy.external.path }}"]
+        command: ["./run.sh"]
         livenessProbe:
           httpGet:
             path: "{{ .Values.proxy.external.path }}/alive"

run.sh

 #!/bin/bash
-find gui/build -type f -not -name "env.js" | xargs -L1 bash -c 'sed "s_/fairdi/nomad/latest/gui_$1/gui_g" $2 > /tmp/temp_file; cp /tmp/temp_file $2;' -- $1