Commit 002427ff authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added gui code to pypi package. Added documentation for bare-metal oasis install.

parent 289c9509
Pipeline #88739 passed with stages
in 24 minutes and 6 seconds
......@@ -3,7 +3,7 @@
**/.mypy_cache
**/__pycache__
**/*.pyc
**/NOMAD.egg-info
**/*.egg-info
.*env/
.pyenv*/
.pytest_cache
......@@ -14,6 +14,8 @@
**/.ipynb_checkpoints
**/.volumes
nomad_lab.egg-info/
data/
local/
target/
......@@ -30,6 +32,7 @@ docs/*.graffle
nomad/normalizing/data/*.db
nomad/normalizing/data/*.msg
nomad/app/static
nomad.yaml
gui/node_modules/
......
......@@ -69,8 +69,6 @@ WORKDIR /install
COPY . /install
RUN python setup.py compile
RUN pip install .[all]
RUN python setup.py sdist
RUN cp dist/nomad-lab-*.tar.gz dist/nomad-lab.tar.gz
RUN python -m nomad.cli dev metainfo > gui/src/metainfo.json
RUN python -m nomad.cli dev search-quantities > gui/src/searchQuantities.json
RUN python -m nomad.cli dev toolkit-metadata > gui/src/toolkitMetadata.json
......@@ -115,24 +113,21 @@ COPY . /app
WORKDIR /app
# transfer installed packages from dependency stage
COPY --from=build /usr/local/lib/python3.7/site-packages /usr/local/lib/python3.7/site-packages
RUN echo "copy 1"
# copy the documentation, its files will be served by the API
COPY --from=build /install/docs/.build /app/docs/.build
RUN echo "copy 2"
# copy the source distribution, its files will be served by the API
COPY --from=build /install/dist /app/dist
RUN echo "copy 3"
# copy the nomad command
COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
RUN echo "copy 4"
# copy the gui
RUN mkdir -p /app/gui
COPY --from=gui_build /app/build /app/gui/build
RUN echo "copy 5"
COPY --from=gui_build /app/build /app/nomad/app/static/gui
# copy the encyclopedia gui production code
COPY --from=gui_build /encyclopedia /app/dependencies/encyclopedia-gui/client
RUN rm -rf /app/dependencies/encyclopedia-gui/client/src
RUN echo "copy 6"
COPY --from=gui_build /encyclopedia /app/nomad/app/static/encyclopedia
# remove the developer config on the gui, will be generated by run.sh from nomad.yaml
RUN rm -f /app/nomad/app/static/gui/env.js
RUN rm -f /app/nomad/app/static/encyclopedia/conf.js
# build the python package dist
RUN python setup.py sdist
RUN cp dist/nomad-lab-*.tar.gz dist/nomad-lab.tar.gz
RUN mkdir -p /app/.volumes/fs
RUN useradd -ms /bin/bash nomad
......
......@@ -6,3 +6,4 @@ include LICENSE.txt
include requirements.txt
include auto_complete_install.sh
include setup.json
recursive-include nomad/app/static *.css *.ico *.html *.json *.js *.map *.txt *.svg *.png
\ No newline at end of file
Subproject commit b512a2ae5acc3e35887c291a59d03f81169e2af3
Subproject commit c378ce3c667fa0b6bd93ecdfd5699b7b9e165fa5
Subproject commit 10973cb74a154c5896633cb50f6e576fddd72852
Subproject commit 56eb84fffa3ae25a4b8dc10aa56da85a4fe6584d
static
\ No newline at end of file
......@@ -19,6 +19,5 @@
from flask import Blueprint
import os.path
gui_folder = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../../dependencies/encyclopedia-gui/client'))
gui_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), 'static/encyclopedia'))
blueprint = Blueprint('encyclopedia', __name__, static_url_path='/', static_folder=gui_folder)
......@@ -19,8 +19,10 @@
from flask import Blueprint, request
import os.path
gui_folder = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../../gui/build'))
gui_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), 'static/gui'))
configuired_gui_folder = os.path.join(gui_folder, '../.gui_configured')
if os.path.exists(configuired_gui_folder):
gui_folder = configuired_gui_folder
blueprint = Blueprint('gui', __name__, static_url_path='/', static_folder=gui_folder)
......
......@@ -612,8 +612,10 @@ def restore(path_to_dump):
@ops.command(help=('Generate an nginx.conf to serve the GUI and proxy pass to API container.'))
@click.option('--prefix', type=str, default='/example_nomad', help='Url path prefix. Default is /example_nomd, can be empty str.')
def nginx_conf(prefix):
@click.option('--prefix', type=str, default=config.services.api_base_path, help='Alter the url path prefix.')
@click.option('--host', type=str, default=config.services.api_host, help='Alter the NOMAD app host.')
@click.option('--port', type=str, default=config.services.api_port, help='Alter the NOMAD port host.')
def nginx_conf(prefix, host, port):
prefix = prefix.rstrip('/')
prefix = '/%s' % prefix.lstrip('/')
......@@ -624,22 +626,22 @@ server {{
proxy_set_header Host $host;
location / {{
proxy_pass http://app:8000;
proxy_pass http://{1}:{2};
}}
location ~ {1}\\/?(gui)?$ {{
rewrite ^ {1}/gui/ permanent;
location ~ {0}\\/?(gui)?$ {{
rewrite ^ {0}/gui/ permanent;
}}
location {1}/gui/ {{
location {0}/gui/ {{
proxy_intercept_errors on;
error_page 404 = @redirect_to_index;
proxy_pass http://app:8000;
proxy_pass http://{1}:{2};
}}
location @redirect_to_index {{
rewrite ^ {1}/gui/index.html break;
proxy_pass http://app:8000;
rewrite ^ {0}/gui/index.html break;
proxy_pass http://{1}:{2};
}}
location ~ \\/gui\\/(service-worker\\.js|meta\\.json)$ {{
......@@ -648,69 +650,37 @@ server {{
if_modified_since off;
expires off;
etag off;
proxy_pass http://app:8000;
proxy_pass http://{1}:{2};
}}
location ~ \\/api\\/uploads\\/?$ {{
client_max_body_size 35g;
proxy_request_buffering off;
proxy_pass http://app:8000;
proxy_pass http://{1}:{2};
}}
location ~ \\/api\\/(raw|archive) {{
proxy_buffering off;
proxy_pass http://app:8000;
proxy_pass http://{1}:{2};
}}
location ~ \\/api\\/mirror {{
proxy_buffering off;
proxy_read_timeout 600;
proxy_pass http://app:8000;
proxy_pass http://{1}:{2};
}}
}}'''.format(prefix))
location ~ \\/encyclopedia\\/ {{
proxy_intercept_errors on;
error_page 404 = @redirect_to_encyclopedia_index;
proxy_pass http://{1}:{2};
}}
@ops.command(help=('Generate a proxy pass config for apache2 reverse proxy servers.'))
@click.option('--prefix', type=str, default='app', help='The path prefix under which everything is proxy passed.')
@click.option('--host', type=str, default='130.183.207.104', help='The host to proxy to.')
@click.option('--port', type=str, default='30001', help='The port to proxy to.')
def apache_conf(prefix, host, port):
print('''\
ProxyPass "/{0}" "http://{1}:{2}/{0}"
ProxyPassReverse "/{0}" "http://{1}:{2}/{0}"
<Proxy http://{1}:{2}/{0}>
ProxyPreserveHost On
<IfModule !mod_access_compat.c>
Require all granted
</IfModule>
<IfModule mod_access_compat.c>
Order allow,deny
Allow from all
</IfModule>
</Proxy>
RequestHeader set "X-Forwarded-Proto" expr=%{{REQUEST_SCHEME}}
RequestHeader set "X-Forwarded-SSL" expr=%{{HTTPS}}
ProxyPass /fairdi/keycloak http://{1}:8002/fairdi/keycloak
ProxyPassReverse /fairdi/keycloak http://{1}:8002/fairdi/keycloak
<Proxy http://{1}:8002/app>
ProxyPreserveHost On
<IfModule !mod_access_compat.c>
Require all granted
</IfModule>
<IfModule mod_access_compat.c>
Order allow,deny
Allow from all
</IfModule>
</Proxy>
RewriteEngine on
RewriteCond %{QUERY_STRING} ^pid=([^&]+)$
RewriteRule ^/NomadRepository-1.1/views/calculation.zul$ /{0}/gui/entry/pid/%1? [R=301]
AllowEncodedSlashes On
'''.format(prefix, host, port)) # type: ignore
location @redirect_to_encyclopedia_index {{
rewrite ^ {0}/encyclopedia/index.html break;
proxy_pass http://{1}:{2};
}}
}}'''.format(prefix, host, port))
@ops.command(help='Updates the AFLOW prototype information using the latest online version and writes the results to a python module in the given FILEPATH.')
......@@ -746,3 +716,76 @@ def update(input_dir, out, verbose):
def ingest(input_path, batch_size, verbose):
from nomad.cli.admin import similarity
similarity.ingest(input_path, batch_size, verbose)
@ops.command(help='Configures the GUIs based on NOMAD config.')
def gui_config():
import os
import os.path
from nomad import config
import glob
import shutil
gui_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../app/static/gui'))
run_gui_folder = os.path.join(gui_folder, '../.gui_configured')
# copy
shutil.rmtree(run_gui_folder, ignore_errors=True)
shutil.copytree(gui_folder, run_gui_folder)
# setup the env
env_js_file = os.path.join(run_gui_folder, 'env.js')
if not os.path.exists(env_js_file):
with open(env_js_file, 'wt') as f:
f.write(('''
window.nomadEnv = {
'appBase': '%s',
'keycloakBase': 'https://nomad-lab.eu/fairdi/keycloak/auth/',
'keycloakRealm': '%s',
'keycloakClientId': '%s',
'debug': false,
'matomoEnabled': false,
'encyclopediaEnabled': true,
'oasis': %s
};''' % (
config.services.api_base_path,
config.keycloak.realm_name,
config.keycloak.client_id,
'true' if config.keycloak.oasis else 'false'
)))
# replace base path in all GUI files
source_file_globs = [
'**/*.json',
'**/*.html',
'**/*.js',
'**/*.js.map',
'**/*.css']
for source_file_glob in source_file_globs:
source_files = glob.glob(os.path.join(run_gui_folder, source_file_glob), recursive=True)
for source_file in source_files:
with open(source_file, 'rt') as f:
file_data = f.read()
file_data = file_data.replace('/fairdi/nomad/latest', config.services.api_base_path)
with open(source_file, 'wt') as f:
f.write(file_data)
gui_folder = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../../app/static/encyclopedia'))
# setup the env
conf_js_file = os.path.join(gui_folder, 'conf.js')
if not os.path.exists(conf_js_file):
with open(conf_js_file, 'wt') as f:
f.write(('''
window.nomadEnv = {
apiRoot: "%s/api/encyclopedia/",
keycloakBase: "%s",
keycloakRealm: "%s",
keycloakClientId: "%s"
};''' % (
config.services.api_base_path,
config.keycloak.server_url,
config.keycloak.realm_name,
config.keycloak.client_id
)))
# Operating an OASIS
The following describes the simplest way to run your own NOMAD.
## What is an OASIS
Originally, NOMAD Central Repository is a service run at Max-Planck's computing facility in Garching, Germany.
However, the NOMAD software is Open-Source, and everybody can run it. Any service that
uses NOMAD software independently is called a *NOMAD OASIS*.
......@@ -11,8 +7,32 @@ uses NOMAD software independently is called a *NOMAD OASIS*.
While several use cases require different setups, this documentation
describes the simplest setup of a NOMAD OASIS. It would allow a group to use NOMAD for
local research data management, while using NOMAD's central user-management and its users.
There are several environment in which you can run a NOMAD OASIS: base-metal linux,
docker with docker-compose, and in a kubernetes cluster. We recommend using docker/docker-compose.
## Before you start
We need to register some information about your OASIS in the central user management:
- The hostname for the machine you run NOMAD on. This is important for redirects between
your OASIS and the central NOMAD user-management and to allow your users to upload files (via GUI or API).
Your machine needs to be accessible under this hostname from the public internet. The host
name needs to be registered with the central NOMAD in order to configure the central user-
management correctly.
- Your NOMAD account that should act as an admin account for your OASIS. This account must be declared
to the central NOMAD as an OASIS admin in order to give it the necessary rights in the central user-
management.
- You will need to know your NOMAD user-id. This information has to be provided by us.
Please [write us](mailto:support@nomad-lab.eu) to register your NOMAD account as an OASIS
admin and to register your hostname. Please replace the indicated configuration items with
the right information.
In principle, you can also run your own user management. This is not yet documented.
## Pre-requisites
## docker and docker compose
### Pre-requisites
NOMAD software is distributed as a set of docker containers. Further, other services
that can be run with docker are required. Further, we use docker-compose to setup
......@@ -25,30 +45,15 @@ database, an **elasticsearch**, a **rabbitmq** distributed task queue, the NOMAD
NOMAD **worker**, and NOMAD **gui**. Refer to this [introduction](/app/docs/introduction.html#architecture)
to learn what each service does and why it is necessary.
There is also some information you need to configure your NOMAD OASIS:
- The hostname for the machine you run NOMAD on. This is important for redirects between
your OASIS and the central NOMAD user-management and to allow your users to upload files (via GUI or API).
Your machine needs to be accessible under this hostname from the public internet. The host
name needs to be registered with the central NOMAD in order to configure the central user-
management correctly.
- A NOMAD account that acts as an admin account for your OASIS. This account must be declared
to the central NOMAD as an OASIS admin in order to give it the necessary rights in the central user-
management.
## Configuration
### Configuration overview
All docker container are configured via docker-compose an the respective `docker-compose.yaml` file.
Further, we will need to mount some configuration files to configure the NOMAD services within
their respective containers.
Please [write us](mailto:support@nomad-lab.eu) to register your NOMAD account as an OASIS
admin and to register your hostname. Please replace the indicated configuration items with
the right information.
There are three files to configure:
- `docker-compose.yaml`
- `nomad.yaml`
- `env.js`
- `nginx.conf`
In this example, we have all files in the same directory (the directory we also work from).
......@@ -134,9 +139,8 @@ services:
- mongo
volumes:
- ./nomad.yaml:/app/nomad.yaml
- ./env.js:/app/gui/build/env.js
- nomad_oasis_files:/app/.volumes/fs
command: ["./run.sh", "/nomad-oasis"]
command: gunicorn -b 0.0.0.0 nomad.app:app
# nomad gui (a reverse proxy for nomad)
gui:
......@@ -202,24 +206,6 @@ You need to change:
A few things to notice:
- Be secretive about your admin credentials; make sure this file is not publicly readable.
### env.js
The GUI also has a config file, called `env.js` with a similar function than `nomad.yaml`.
```js
window.nomadEnv = {
'appBase': '/nomad-oasis/',
'keycloakBase': 'https://nomad-lab.eu/fairdi/keycloak/auth/',
'keycloakRealm': 'fairdi_nomad_prod',
'keycloakClientId': 'nomad_public',
'debug': false,
'oasis': true
};
```
You need to change:
- `appBase` defines the base path again. It needs to be changed, if you use a different base path.
### nginx.conf
The GUI container serves as a proxy that forwards request to the app container. The
......@@ -293,7 +279,7 @@ Gunicorn is the WSGI-server that runs the nomad app. Consult the
[gunicorn documentation](https://docs.gunicorn.org/en/stable/configure.html) for
configuration options.
## Starting and stopping
### Starting and stopping NOMAD
If you prepared the above files, simply use the usual `docker-compose` commands to start everything.
......@@ -349,8 +335,116 @@ If you want to report problems with your OASIS. Please provide the logs for
- nomad_oasis_worker
- nomad_oasis_gui
## base linux (without docker)
### Pre-requisites
We will run NOMAD from the *nomad-lab* Python package. This package contains all the necessary
code for running the processing, api, and gui. But, it is missing the necessary databases. You might be
able to run NOMAD in user space.
You will need:
- preferably a linux computer, which Python 3.7, preferable a Python virtual environment
- elasticsearch 6.x, running without users and authentication, preferable on the default settings
- mongodb 4.x, running without users and authentication, preferable on the default settings
- rabbitmq 3.x, running without users and authentication, preferable on the default settings
- nginx
- an empty directory to work in
### Install the NOMAD Python package
```sh
virtualenv -p `which python3` nomadpyenv
source nomadpyenv/bin/activate
```
```sh
pip install nomad-lab[all]
```
```sh
curl "https://nomad-lab.eu/prod/rae/beta/dist/nomad-lab.tar.gz" -o nomad-lab.tar.gz
pip install nomad-lab.tar.gz[all]
```
### Configure NOMAD - nomad.yaml
```yaml
client:
url: 'http://<your-host>/nomad-oasis/api'
services:
api_base_path: '/nomad-oasis'
admin_user_id: '<your admin user id>'
keycloak:
realm_name: fairdi_nomad_prod
username: '<your admin username>'
password: '<your admin user password>'
oasis: true
mongo:
db_name: nomad_v0_8
elastic:
index_name: nomad_v0_8
```
You need to change:
- Replace `your-host` and admin credentials respectively.
- `api_base_path` defines the path under with the app is run. It needs to be changed, if you use a different base path.
A few things to notice:
- Be secretive about your admin credentials; make sure this file is not publicly readable.
### Configure NOMAD - nginx
You can generate a suitable `nginx.conf` with the `nomad` command line command that
comes with the *nomad-lab* Python package. If you server other content but NOMAD with
your nginx, you need to incorporate the config accordingly.
If you have a standard installation of nginx, this might work. Adapt to your set-up:
```sh
nomad admin ops nginx-conf > /etc/nginx/conf.d/default.conf
nginx -t
nginx -s reload
```
If you want to run nginx in docker, this might work. Adapt to your set-up:
```sh
nomad admin ops nginx-conf --host host.docker.internal > nginx.conf
docker run --rm -v `pwd`/nginx.conf:/etc/nginx/conf.d/default.conf -p 80:80 nginx:stable nginx -g 'daemon off;' &
```
### Running NOMAD
Before you start, we need to transfer your `nomad.yaml` config values to the GUI's
javascript. You need to repeat this, if you change your `nomad.yaml`. You can do this by running:
```
nomad admin ops gui-config
```
To run NOMAD, you must run two services. One is the NOMAD app, it serves the API and GUI:
```
gunicorn "${params[@]}" -b 0.0.0.0:8000 nomad.app:app
```
And the NOMAD worker, that runs the NOMAD processing.
```
celery worker -l info -A nomad.processing -Q celery,calcs,uploads
```
This should give you a working OASIS at `http://<your-host>/<your-path-prefix>`.
## kubernetes
*This is not yet documented.*
## Migrating from an older version (0.7.x to 0.8.x)
*This documentation is outdated and will be removed in future releases.*
Between versions 0.7.x and 0.8.x we needed to change how archive and metadata data is stored
internally in files and databases. This means you cannot simply start a new version of
NOMAD on top of the old data. But there is a strategy to adapt the data.
......@@ -361,7 +455,7 @@ docker-compose stop
docker-compose rm -f
```
Update you config files (`docker-compose.yaml`, `nomad.yaml`, `env.js`, `nginx.conf`) according
Update you config files (`docker-compose.yaml`, `nomad.yaml`, `nginx.conf`) according
to the latest documentation (see above). Especially make sure to select a new name for
databases and search index in your `nomad.yaml` (e.g. `nomad_v0_8`). This will
allow us to create new data while retaining the old, i.e. to copy the old data over.
......
......@@ -72,9 +72,8 @@ services:
- mongo
volumes:
- ./nomad.yaml:/app/nomad.yaml
- ./env.js:/app/gui/build/env.js
- nomad_oasis_files:/app/.volumes/fs
command: ["./run.sh", "/nomad-oasis"]
command: ./run.sh
# nomad gui (a reverse proxy for nomad)
gui:
......
window.nomadEnv = {
'appBase': '/nomad-oasis/',
'keycloakBase': 'https://nomad-lab.eu/fairdi/keycloak/auth/',
'keycloakRealm': 'fairdi_nomad_prod',
'keycloakClientId': 'nomad_public',
'debug': false,
'matomoEnabled': false,
'encyclopediaEnabled': false,
'oasis': true
};
\ No newline at end of file
......@@ -47,4 +47,15 @@ server {
proxy_read_timeout 600;
proxy_pass http://app:8000;
}
location ~ \/encyclopedia\/ {
proxy_intercept_errors on;
error_page 404 = @redirect_to_encyclopedia_index;
proxy_pass http://app:8000;
}
location @redirect_to_encyclopedia_index {
rewrite ^ /nomad-oasis/encyclopedia/index.html break;
proxy_pass http://app:8000;
}
}
\ No newline at end of file
......@@ -168,11 +168,11 @@ spec:
name: staging-volume
- mountPath: /nomad
name: nomad-volume
- mountPath: /app/gui/build/env.js
- mountPath: /app/nomad/static/gui/env.js
readOnly: true
subPath: env.js
name: gui-env-js
- mountPath: /app/dependencies/encyclopedia-gui/client/conf.js
- mountPath: /app/nomad/static/encyclopedia/conf.js
readOnly: true
subPath: conf.js
name: enc-conf-js
......@@ -223,7 +223,7 @@ spec:
name: {{ .Values.datacite.secret }}
key: user
{{ end }}
command: ["./run.sh", "{{ .Values.proxy.external.path }}"]
command: ["./run.sh"]
livenessProbe:
httpGet:
path: "{{ .Values.proxy.external.path }}/alive"
......
#!/bin/bash
find gui/build -type f -not -name "env.js" | xargs -L1 bash -c 'sed "s_/fairdi/nomad/latest/gui_$1/gui_g" $2 > /tmp/temp_file; cp /tmp/temp_file $2;' -- $1
python -m nomad.cli admin ops gui-config