Added deploy to ci. Fixed integration test, swagger after refactor. Unified helm.

.dockerignore

 **/*.git
+**/.cache
 **/.mypy_cache
 **/__pycache__
 **/*.pyc
@@ -7,13 +8,30 @@
 .pytest_cache
 .vscode/
 .volumes/
+.git/
+.coverage
+
+data/
+local/
+target/
+examples/
+ops/
+
 dependencies/**/test
 dependencies/**/tests
-data/
+dependencies/**/regtests
+dependencies/parsers/phonopy-library/example*
+
 docs/.build
 docs/*.graffle
-.coverage
-examples/
-local/
-target/
+
+nomad/normalizing/data/*.db
+nomad/normalizing/data/*.msg
+nomad.yaml
+
 gui/node_modules/
+gui/build/
+gui/public/metainfo/
+gui/npm-debug.log*
+gui/yarn-debug.log*
+gui/yarn-error.log*
\ No newline at end of file

.gitignore

@@ -22,5 +22,5 @@ target/
 .vscode/
 vscode/
 nomad.yaml
-gunicorn.log.conf
-gunicorn.conf
+./gunicorn.log.conf
+./gunicorn.conf

.gitlab-ci.yml

 # default installed image for docker executor is: python:3.6
 # using an image that can do git, docker, docker-compose
-image: youpy/docker-compose-git
+image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
 
 # Uncomment the next lines, to run each pipline/job in its own docker environment.
 # Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...).
@@ -12,19 +12,14 @@ image: youpy/docker-compose-git
 stages:
   - build
   - test
-  - integration
-  - release
   - deploy
+  - release
+
 
 variables:
-  TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:test_${CI_COMMIT_REF_NAME}
-  RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
+  TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
   STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
   LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
-  FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:test_${CI_COMMIT_REF_NAME}
-  FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
-  FRONTEND_STABLE_IMAGE:  gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:stable
-  FRONTEND_LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
   KUBECONFIG: /etc/deploy/config
 
 build:
@@ -32,6 +27,7 @@ build:
   before_script:
     - git submodule sync
     - git submodule update --init
+    # create the version information
     - ./gitinfo.sh
   script:
     # ignore test directories of dependencies, there is a lot of data that we not use
@@ -44,19 +40,6 @@ build:
     - /^dev-.*$/
     - tags
 
-
-buildgui:
-  stage: build
-  script:
-    - cd gui
-    - ./version.sh
-    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
-    - docker build -t $FRONTEND_TEST_IMAGE .
-    - docker push $FRONTEND_TEST_IMAGE
-  except:
-    - /^dev-.*$/
-    - tags
-
 linting:
   stage: test
   image: $TEST_IMAGE
@@ -103,6 +86,21 @@ tests:
       - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
       - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
 
+deploy:
+  stage: deploy
+  before_script:
+    - mkdir -p /etc/deploy
+    - echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
+  script:
+    - helm dependency update ops/helm/nomad
+    - helm upgrade --namespace nomad --install $CI_COMMIT_REF_NAME ops/helm/nomad -f ops/helm/nomad/ci-dev-values.yaml --set proxy.external.path=/dev/nomad/$CI_COMMIT_REF_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
+    - docker pull $TEST_IMAGE
+    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n $CI_DEV_CLUSTER_PROXY/dev/nomad/$CI_COMMIT_REF_NAME/api -u admin -w $CI_NOMAD_ADMIN_PASSWORD integrationtests --skip-doi
+
+  except:
+    - /^dev-.*$/
+  when: manual
+
 release:
   stage: release
   script:
@@ -110,9 +108,7 @@ release:
     - docker pull $TEST_IMAGE
     - docker tag $TEST_IMAGE $LATEST_IMAGE
     - docker push $LATEST_IMAGE
-    - docker pull $FRONTEND_TEST_IMAGE
-    - docker tag $FRONTEND_TEST_IMAGE $FRONTEND_LATEST_IMAGE
-    - docker push $FRONTEND_LATEST_IMAGE
+
   except:
     - /^dev-.*$/
   when: manual
@@ -122,35 +118,9 @@ release_version:
   script:
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
     - docker pull $LATEST_IMAGE
-    - docker tag $LATEST_IMAGE $RELEASE_IMAGE
+    - docker tag $LATEST_IMAGE $CI_COMMIT_TAG
     - docker tag $LATEST_IMAGE $STABLE_IMAGE
-    - docker push $RELEASE_IMAGE
     - docker push $STABLE_IMAGE
-    - docker pull $FRONTEND_LATEST_IMAGE
-    - docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_RELEASE_IMAGE
-    - docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_STABLE_IMAGE
-    - docker push $FRONTEND_RELEASE_IMAGE
-    - docker push $FRONTEND_STABLE_IMAGE
+    - docker push $CI_COMMIT_TAG
   only:
     - tags
-
-## TODO This has to be fixed, we need one for testing, staging, production
-# deploy_coe_staging:
-#   stage: deploy
-#   image: dtzar/helm-kubectl
-#   before_script:
-#     - mkdir -p /etc/deploy
-#     # kube_config is a CI/CD variable set in GitLab GUI
-#     - echo $CI_KUBE_CONFIG | base64 -d > /etc/deploy/config
-#     - helm init --upgrade
-#     - helm repo add stable https://kubernetes-charts.storage.googleapis.com/
-#     - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
-#     - helm repo update
-#   script:
-#     - cd ops/helm/nomad
-#     - export KUBECONFIG=/etc/deploy/config
-#     - helm dep build
-#     - helm upgrade nomad-coe-staging . --recreate-pods;
-#   except:
-#     - /^dev-.*$/
-#   when: manual

Dockerfile

@@ -23,7 +23,7 @@
 # We use slim for the final image
 FROM python:3.6-slim as final
 
-# First, build everything in a build image
+# First, build all python stuff in a python build image
 FROM python:3.6-stretch as build
 RUN mkdir /install
 
@@ -71,10 +71,21 @@ RUN \
     find /usr/local/lib/python3.6/ -name 'test' -exec rm -r '{}' + && \
     find /usr/local/lib/python3.6/site-packages/ -name '*.so' -print -exec sh -c 'file "{}" | grep -q "not stripped" && strip -s "{}"' \;
 
-# Second, create a slim final image
+# Second built the GUI in a gui build image
+FROM node:latest as gui_build
+RUN mkdir -p /app
+WORKDIR /app
+ENV PATH /app/node_modules/.bin:$PATH
+COPY gui/package.json /app/package.json
+COPY gui/yarn.lock /app/yarn.lock
+RUN yarn
+COPY gui /app
+RUN yarn run build
+
+# Third, create a slim final image
 FROM final
 
-RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev
+RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev curl
 
 # copy the sources for tests, coverage, qa, etc.
 COPY . /app
@@ -89,12 +100,17 @@ RUN echo "copy 2"
 COPY --from=build /install/docs/.build /app/docs/.build
 RUN echo "copy 3"
 # copy the nomad command
-COPY --from=build /usr/local/bin/nomad /usr/local/bin/nomad
+COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
 RUN echo "copy 4"
+# copy the gui
+RUN mkdir -p /app/gui
+COPY --from=gui_build /app/build /app/gui/build
+RUN echo "copy 5"
 
 RUN mkdir -p /app/.volumes/fs
 RUN useradd -ms /bin/bash nomad
 RUN chown -R nomad /app
+RUN chmod a+rx run.sh
 USER nomad
 
 VOLUME /app/.volumes/fs

docs/depl_docker.rst

-.. mdinclude:: ../ops/docker-compose/nomad/README.md
+.. mdinclude:: ../ops/docker-compose/infrastructure/README.md

docs/setup.md

@@ -131,39 +131,31 @@ must run to make use of nomad. We use *docker* and *docker-compose* to create a
 unified environment that is easy to build and to run.
 
 You can use *docker* to run all necessary 3rd-party components and run all nomad
-services manually from your python environment. Or you can run everything within
-docker containers. The former is often preferred during development, since it allows
+services manually from your python environment. You can also run nomad in docker,
+but using Python is often preferred during development, since it allows
 you change things, debug, and re-run things quickly. The later one brings you
-closer to the environment that will be used to run nomad in production.
+closer to the environment that will be used to run nomad in production. For
+development we recommend to skip the next step.
 
 ### Docker images for nomad
-There are currently two different images and respectively two different docker files:
-`Dockerfile`, and `gui/Dockerfile`.
-
 Nomad comprises currently two services,
 the *worker* (does the actual processing), and the *app*. Those services can be
 run from one image that have the nomad python code and all dependencies installed. This
-is covered by the `Dockerfile`.
-
-The gui is served via containers based on the `gui/Dockerfile` which contains the
-react-js frontend code. Before this image can be build, make sure to execute
+is covered by the `Dockerfile` in the root directory
+of the nomad sources. The gui is served also served from the *app* which entails the react-js frontend code.
 
+Before building the image, make sure to execute
 ```
-cd gui
 ./gitinfo.sh
-cd ..
 ```
-
-This allows to gui to present some information about the current git revision without
+This allows the app to present some information about the current git revision without
 having to copy the git itself to the docker build context.
 
-The images are build via *docker-compose* and don't have to be created manually.
-
 ### Run necessary 3-rd party services with docker-compose
 
 You can run all containers with:
 ```
-cd ops/docker-compose/nomad
+cd ops/docker-compose/infrastructure
 docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d mongo elastic rabbitmq
 ```
 

gitinfo.sh

 #!/bin/sh
-echo log, ref, version, commit = \"$(git log -1 --oneline)\", \"$(git describe --all)\", \"$(git describe --tags)\", \"$(git rev-parse --verify --short HEAD)\" > nomad/gitinfo.py
\ No newline at end of file
+# python/backend
+echo log, ref, version, commit = \"$(git log -1 --oneline)\", \"$(git describe --all)\", \"$(git describe --tags)\", \"$(git rev-parse --verify --short HEAD)\" > nomad/gitinfo.py
+
+# gui
+commit=`git rev-parse --short --verify HEAD`
+sed -i -e "s/nomad-gui-commit-placeholder/$commit/g" gui/package.json
+rm -f gui/package.json-e
\ No newline at end of file

gui/.dockerignore

-.vscode/
-.git/
-node_modules/
-build/
-public/metainfo/
-
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
\ No newline at end of file

gui/run.sh

-#!/bin/sh
-
-find . -type f -exec sed -i "s_/fairdi/nomad/latest/gui_$1/gui_g" {} \;
-nginx -g "daemon off;"
\ No newline at end of file

gui/version.sh

-#!/bin/sh
-commit=`git rev-parse --short --verify HEAD`
-sed -i -e "s/nomad-gui-commit-placeholder/$commit/g" package.json
-rm -f package.json-e

nomad/app/__init__.py

@@ -34,6 +34,7 @@ from nomad import config, utils as nomad_utils
 from .api import blueprint as api_blueprint, api
 from .optimade import blueprint as optimade_blueprint, api as optimade
 from .docs import blueprint as docs_blueprint
+from .gui import blueprint as gui_blueprint
 from . import common
 
 
@@ -91,6 +92,7 @@ CORS(app)
 app.register_blueprint(api_blueprint, url_prefix='/api')
 app.register_blueprint(optimade_blueprint, url_prefix='/optimade')
 app.register_blueprint(docs_blueprint, url_prefix='/docs')
+app.register_blueprint(gui_blueprint, url_prefix='/gui')
 
 
 @app.errorhandler(Exception)

nomad/app/api/archive.py

@@ -214,7 +214,7 @@ class ArchiveDownloadResource(Resource):
 
 
 _archive_query_model = api.inherit('ArchiveSearch', search_model, {
-    'query': fields.Nested(query_model, description='The query used to find the requested entries.'),
+    'query': fields.Nested(query_model, description='The query used to find the requested entries.', skip_none=True),
     'query_schema': fields.Raw(description='The query schema that defines what archive data to retrive.')
 })
 
@@ -225,9 +225,8 @@ class ArchiveQueryResource(Resource):
     @api.response(400, 'Invalid requests, e.g. wrong owner type or bad search parameters')
     @api.response(401, 'Not authorized to access the data.')
     @api.response(404, 'The upload or calculation does not exist')
-    @api.response(200, 'Archive data send')
     @api.expect(_archive_query_model)
-    @api.marshal_with(_archive_query_model, skip_none=True, code=200, description='Search results sent')
+    @api.marshal_with(_archive_query_model, skip_none=True, code=200, description='Archive search results sent')
     @authenticate()
     def post(self):
         '''

nomad/app/api/common.py

@@ -78,7 +78,7 @@ search_model_fields = {
 search_model = api.model('Search', search_model_fields)
 
 query_model_fields = {
-    qualified_name: fields.Raw(description=quantity.description)
+    qualified_name: quantity.flask_field
     for qualified_name, quantity in search.search_quantities.items()}
 
 query_model_fields.update(**{

nomad/app/api/dataset.py

@@ -34,7 +34,7 @@ ns = api.namespace(
 
 dataset_model = generate_flask_restplus_model(api, Dataset.m_def)
 dataset_list_model = api.model('DatasetList', {
-    'pagination': fields.Nested(model=pagination_model),
+    'pagination': fields.Nested(model=pagination_model, skip_none=True),
     'results': fields.List(fields.Nested(model=dataset_model, skip_none=True))
 })
 

nomad/app/api/upload.py

@@ -80,7 +80,7 @@ upload_model = api.inherit('UploadProcessing', proc_model, {
 })
 
 upload_list_model = api.model('UploadList', {
-    'pagination': fields.Nested(model=pagination_model),
+    'pagination': fields.Nested(model=pagination_model, skip_none=True),
     'results': fields.List(fields.Nested(model=upload_model, skip_none=True))
 })
 
@@ -103,7 +103,7 @@ upload_with_calcs_model = api.inherit('UploadWithPaginatedCalculations', upload_
         'pagination': fields.Nested(model=api.inherit('UploadCalculationPagination', pagination_model, {
             'successes': fields.Integer,
             'failures': fields.Integer,
-        })),
+        }), skip_none=True),
         'results': fields.List(fields.Nested(model=calc_model, skip_none=True))
     }), skip_none=True)
 })

gui/Dockerfile → nomad/app/gui.py

@@ -12,33 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This docker image is used for the GUI container that serves the static GUI
-# files.
-#
-# This is a multistage build, one stage for building the GUI and one final stage
-# intended for the actual GUI container that serves the GUI.
-
-# build environment
-FROM node:latest as build
-RUN mkdir -p /nomad/app
-WORKDIR /nomad/app
-ENV PATH /nomad/app/node_modules/.bin:$PATH
-COPY package.json /nomad/app/package.json
-COPY yarn.lock /nomad/app/yarn.lock
-RUN yarn
-COPY . /nomad/app
-
-RUN yarn run build
-
-# production environment
-FROM nginx:1.13.9-alpine
-COPY --from=build /nomad/app/build /app/nomad
-
-WORKDIR /app/nomad
-COPY ./run.sh run.sh
-RUN chmod a+rx run.sh
-CMD nginx -g "daemon off;"
-
-VOLUME /etc/nginx/conf.d
+from flask import Blueprint
+import os.path
 
-EXPOSE 8080/tcp
+gui_folder = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '../../gui/build'))
+blueprint = Blueprint('gui', __name__, static_url_path='/', static_folder=gui_folder)

nomad/cli/admin/admin.py

@@ -229,59 +229,53 @@ def nginx_conf(prefix):
 server {{
     listen        80;
     server_name   www.example.com;
+    proxy_set_header Host $host;
 
-    location /{0} {{
-        return 301 /example-nomad/gui;
+    location / {{
+        proxy_pass http://app:8000;
     }}
 
-    location {1}/gui {{
-        root      /app/;
-        rewrite ^{1}/gui/(.*)$ /nomad/$1 break;
-        try_files $uri {1}/gui/index.html;
+    location ~ {1}\\/?(gui)?$ {{
+        rewrite ^ {1}/gui/ permanent;
     }}
 
-    location {1}/gui/service-worker.js {{
-        add_header Last-Modified $date_gmt;
-        add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
-        if_modified_since off;
-        expires off;
-        etag off;
-        root      /app/;
-        rewrite ^{1}/gui/service-worker.js /nomad/service-worker.js break;
+    location {1}/gui/ {{
+        proxy_intercept_errors on;
+        error_page 404 = @redirect_to_index;
+        proxy_pass http://app:8000;
+    }}
+
+    location @redirect_to_index {{
+        rewrite ^ {1}/gui/index.html break;
+        proxy_pass http://app:8000;
     }}
 
-    location {1}/gui/meta.json {{
+    location ~ \\/gui\\/(service-worker\\.js|meta\\.json)$ {{
         add_header Last-Modified $date_gmt;
         add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
         if_modified_since off;
         expires off;
         etag off;
-        root      /app/;
-        rewrite ^{1}/gui/meta.json /nomad/meta.json break;
-    }}
-
-    location {1}/api {{
-        proxy_set_header Host $host;
-        proxy_pass_request_headers on;
-        proxy_pass http://api:8000;
+        proxy_pass http://app:8000;
     }}
 
-    location {1}/api/uploads {{
+    location ~ \\/api\\/uploads\\/?$ {{
         client_max_body_size 35g;
         proxy_request_buffering off;
-        proxy_set_header Host $host;
-        proxy_pass_request_headers on;
-        proxy_pass http://api:8000;
+        proxy_pass http://app:8000;
+    }}
+
+    location ~ \\/api\\/(raw|archive) {{
+        proxy_buffering off;
+        proxy_pass http://app:8000;
     }}
 
-    location {1}/api/raw {{
+    location ~ \\/api\\/mirror {{
         proxy_buffering off;
-        proxy_set_header Host $host;
-        proxy_pass_request_headers on;
-        proxy_pass http://api:8000;
+        proxy_read_timeout 600;
+        proxy_pass http://app:8000;
     }}
-}}
-    '''.format(prefix.lstrip('/'), prefix))
+}}'''.format(prefix))
 
 
 @ops.command(help=('Generate a proxy pass config for apache2 reverse proxy servers.'))

nomad/cli/admin/run.py

@@ -43,7 +43,7 @@ def run_app(**kwargs):
     from nomad import infrastructure
     from nomad.app.__main__ import run_dev_server
     infrastructure.setup()
-    run_dev_server(port=8000, **kwargs)
+    run_dev_server(port=config.services.api_port, **kwargs)
 
 
 def run_worker():

nomad/cli/client/integrationtests.py

@@ -123,11 +123,22 @@ def integrationtests(ctx, skip_parsers, skip_publish, skip_doi, skip_mirror):
         assert len(search.results) <= search.pagination.total
 
         print('performing archive paginated search')
-        result = client.archive.archive_query(page=1, per_page=10, **query).response().result
+        result = client.archive.post_archive_query(payload={
+            'pagination': {
+                'page': 1,
+                'per_page': 10
+            },
+            'query': query
+        }).response().result
         assert len(result.results) > 0
 
         print('performing archive scrolled search')
-        result = client.archive.archive_query(scroll=True, **query).response().result
+        result = client.archive.post_archive_query(payload={
+            'scroll': {
+                'scroll': True
+            },
+            'query': query
+        }).response().result
         assert len(result.results) > 0
 
         print('performing download')

nomad/infrastructure.py

@@ -106,8 +106,6 @@ def setup_elastic():
     try:
         from nomad.search import entry_document
         entry_document.init(index=config.elastic.index_name)
-        entry_document._index._name = config.elastic.index_name
-        logger.info('initialized elastic index', index_name=config.elastic.index_name)
     except RequestError as e:
         if e.status_code == 400 and 'resource_already_exists_exception' in e.error:
             # happens if two services try this at the same time
@@ -115,6 +113,9 @@ def setup_elastic():
         else:
             raise e
 
+    entry_document._index._name = config.elastic.index_name
+    logger.info('initialized elastic index', index_name=config.elastic.index_name)
+
     return elastic_client