diff --git a/.dockerignore b/.dockerignore index c004b33d7e7aed23d66ab7fb62b17d0f2dea5be1..3ddbd054de5baff3c948a411d52076beb223203f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,5 @@ **/*.git +**/.cache **/.mypy_cache **/__pycache__ **/*.pyc @@ -7,13 +8,30 @@ .pytest_cache .vscode/ .volumes/ +.git/ +.coverage + +data/ +local/ +target/ +examples/ +ops/ + dependencies/**/test dependencies/**/tests -data/ +dependencies/**/regtests +dependencies/parsers/phonopy-library/example* + docs/.build docs/*.graffle -.coverage -examples/ -local/ -target/ + +nomad/normalizing/data/*.db +nomad/normalizing/data/*.msg +nomad.yaml + gui/node_modules/ +gui/build/ +gui/public/metainfo/ +gui/npm-debug.log* +gui/yarn-debug.log* +gui/yarn-error.log* \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1b0c8e5da37ea85af82a2627925000ef74bbe228..8429152463f63bfdf0e375d2f487402b9ad5b0fd 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,5 @@ target/ .vscode/ vscode/ nomad.yaml -gunicorn.log.conf -gunicorn.conf +./gunicorn.log.conf +./gunicorn.conf diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 189dc975f6277e964370e78f416d044d55b7f0db..0cde5bbe465aa90ef84eb78e8f4e33060f4804b4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ # default installed image for docker executor is: python:3.6 # using an image that can do git, docker, docker-compose -image: youpy/docker-compose-git +image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner # Uncomment the next lines, to run each pipline/job in its own docker environment. # Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...). @@ -12,19 +12,14 @@ image: youpy/docker-compose-git stages: - build - test - - integration - - release - deploy + - release + variables: - TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:test_${CI_COMMIT_REF_NAME} - RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME} + TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME} STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest - FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:test_${CI_COMMIT_REF_NAME} - FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME} - FRONTEND_STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:stable - FRONTEND_LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest KUBECONFIG: /etc/deploy/config build: @@ -32,6 +27,7 @@ build: before_script: - git submodule sync - git submodule update --init + # create the version information - ./gitinfo.sh script: # ignore test directories of dependencies, there is a lot of data that we not use @@ -44,19 +40,6 @@ build: - /^dev-.*$/ - tags - -buildgui: - stage: build - script: - - cd gui - - ./version.sh - - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de - - docker build -t $FRONTEND_TEST_IMAGE . - - docker push $FRONTEND_TEST_IMAGE - except: - - /^dev-.*$/ - - tags - linting: stage: test image: $TEST_IMAGE @@ -103,6 +86,21 @@ tests: - $CI_COMMIT_REF_NAME =~ /^dev-.*$/ - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i +deploy: + stage: deploy + before_script: + - mkdir -p /etc/deploy + - echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG} + script: + - helm dependency update ops/helm/nomad + - helm upgrade --namespace nomad --install $CI_COMMIT_REF_NAME ops/helm/nomad -f ops/helm/nomad/ci-dev-values.yaml --set proxy.external.path=/dev/nomad/$CI_COMMIT_REF_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait + - docker pull $TEST_IMAGE + - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n $CI_DEV_CLUSTER_PROXY/dev/nomad/$CI_COMMIT_REF_NAME/api -u admin -w $CI_NOMAD_ADMIN_PASSWORD integrationtests --skip-doi + + except: + - /^dev-.*$/ + when: manual + release: stage: release script: @@ -110,9 +108,7 @@ release: - docker pull $TEST_IMAGE - docker tag $TEST_IMAGE $LATEST_IMAGE - docker push $LATEST_IMAGE - - docker pull $FRONTEND_TEST_IMAGE - - docker tag $FRONTEND_TEST_IMAGE $FRONTEND_LATEST_IMAGE - - docker push $FRONTEND_LATEST_IMAGE + except: - /^dev-.*$/ when: manual @@ -122,35 +118,9 @@ release_version: script: - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de - docker pull $LATEST_IMAGE - - docker tag $LATEST_IMAGE $RELEASE_IMAGE + - docker tag $LATEST_IMAGE $CI_COMMIT_TAG - docker tag $LATEST_IMAGE $STABLE_IMAGE - - docker push $RELEASE_IMAGE - docker push $STABLE_IMAGE - - docker pull $FRONTEND_LATEST_IMAGE - - docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_RELEASE_IMAGE - - docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_STABLE_IMAGE - - docker push $FRONTEND_RELEASE_IMAGE - - docker push $FRONTEND_STABLE_IMAGE + - docker push $CI_COMMIT_TAG only: - tags - -## TODO This has to be fixed, we need one for testing, staging, production -# deploy_coe_staging: -# stage: deploy -# image: dtzar/helm-kubectl -# before_script: -# - mkdir -p /etc/deploy -# # kube_config is a CI/CD variable set in GitLab GUI -# - echo $CI_KUBE_CONFIG | base64 -d > /etc/deploy/config -# - helm init --upgrade -# - helm repo add stable https://kubernetes-charts.storage.googleapis.com/ -# - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/ -# - helm repo update -# script: -# - cd ops/helm/nomad -# - export KUBECONFIG=/etc/deploy/config -# - helm dep build -# - helm upgrade nomad-coe-staging . --recreate-pods; -# except: -# - /^dev-.*$/ -# when: manual diff --git a/Dockerfile b/Dockerfile index 7c9defe123507ae6acea18a0635996ccc6f20233..d74ab515b9fcb14fe8d91ae60f45914845d2b558 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ # We use slim for the final image FROM python:3.6-slim as final -# First, build everything in a build image +# First, build all python stuff in a python build image FROM python:3.6-stretch as build RUN mkdir /install @@ -71,10 +71,21 @@ RUN \ find /usr/local/lib/python3.6/ -name 'test' -exec rm -r '{}' + && \ find /usr/local/lib/python3.6/site-packages/ -name '*.so' -print -exec sh -c 'file "{}" | grep -q "not stripped" && strip -s "{}"' \; -# Second, create a slim final image +# Second built the GUI in a gui build image +FROM node:latest as gui_build +RUN mkdir -p /app +WORKDIR /app +ENV PATH /app/node_modules/.bin:$PATH +COPY gui/package.json /app/package.json +COPY gui/yarn.lock /app/yarn.lock +RUN yarn +COPY gui /app +RUN yarn run build + +# Third, create a slim final image FROM final -RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev +RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev curl # copy the sources for tests, coverage, qa, etc. COPY . /app @@ -89,12 +100,17 @@ RUN echo "copy 2" COPY --from=build /install/docs/.build /app/docs/.build RUN echo "copy 3" # copy the nomad command -COPY --from=build /usr/local/bin/nomad /usr/local/bin/nomad +COPY --from=build /usr/local/bin/nomad /usr/bin/nomad RUN echo "copy 4" +# copy the gui +RUN mkdir -p /app/gui +COPY --from=gui_build /app/build /app/gui/build +RUN echo "copy 5" RUN mkdir -p /app/.volumes/fs RUN useradd -ms /bin/bash nomad RUN chown -R nomad /app +RUN chmod a+rx run.sh USER nomad VOLUME /app/.volumes/fs diff --git a/docs/depl_docker.rst b/docs/depl_docker.rst index 8595e4871e1c437ab09bbde767a0ad97dcf1169d..249be5815ef67c85acde001d1b84de4a33975f5c 100644 --- a/docs/depl_docker.rst +++ b/docs/depl_docker.rst @@ -1 +1 @@ -.. mdinclude:: ../ops/docker-compose/nomad/README.md +.. mdinclude:: ../ops/docker-compose/infrastructure/README.md diff --git a/docs/setup.md b/docs/setup.md index b259174b881d64d96fa34e55b165612e1223b351..6dd6fc53f15fdc8d5a86854de27d687c0620ed4c 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -131,39 +131,31 @@ must run to make use of nomad. We use *docker* and *docker-compose* to create a unified environment that is easy to build and to run. You can use *docker* to run all necessary 3rd-party components and run all nomad -services manually from your python environment. Or you can run everything within -docker containers. The former is often preferred during development, since it allows +services manually from your python environment. You can also run nomad in docker, +but using Python is often preferred during development, since it allows you change things, debug, and re-run things quickly. The later one brings you -closer to the environment that will be used to run nomad in production. +closer to the environment that will be used to run nomad in production. For +development we recommend to skip the next step. ### Docker images for nomad -There are currently two different images and respectively two different docker files: -`Dockerfile`, and `gui/Dockerfile`. - Nomad comprises currently two services, the *worker* (does the actual processing), and the *app*. Those services can be run from one image that have the nomad python code and all dependencies installed. This -is covered by the `Dockerfile`. - -The gui is served via containers based on the `gui/Dockerfile` which contains the -react-js frontend code. Before this image can be build, make sure to execute +is covered by the `Dockerfile` in the root directory +of the nomad sources. The gui is served also served from the *app* which entails the react-js frontend code. +Before building the image, make sure to execute ``` -cd gui ./gitinfo.sh -cd .. ``` - -This allows to gui to present some information about the current git revision without +This allows the app to present some information about the current git revision without having to copy the git itself to the docker build context. -The images are build via *docker-compose* and don't have to be created manually. - ### Run necessary 3-rd party services with docker-compose You can run all containers with: ``` -cd ops/docker-compose/nomad +cd ops/docker-compose/infrastructure docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d mongo elastic rabbitmq ``` diff --git a/gitinfo.sh b/gitinfo.sh index bbaf0cc3902109f3ef97a180317c23639f5bdb02..9dda3f9ecb1ebb5b7853ee7644382fc4a7f1c09f 100755 --- a/gitinfo.sh +++ b/gitinfo.sh @@ -1,2 +1,8 @@ #!/bin/sh -echo log, ref, version, commit = \"$(git log -1 --oneline)\", \"$(git describe --all)\", \"$(git describe --tags)\", \"$(git rev-parse --verify --short HEAD)\" > nomad/gitinfo.py \ No newline at end of file +# python/backend +echo log, ref, version, commit = \"$(git log -1 --oneline)\", \"$(git describe --all)\", \"$(git describe --tags)\", \"$(git rev-parse --verify --short HEAD)\" > nomad/gitinfo.py + +# gui +commit=`git rev-parse --short --verify HEAD` +sed -i -e "s/nomad-gui-commit-placeholder/$commit/g" gui/package.json +rm -f gui/package.json-e \ No newline at end of file diff --git a/gui/.dockerignore b/gui/.dockerignore deleted file mode 100644 index 8acd7dfe07e6a34f01b62f8e3a8e370738d7cf81..0000000000000000000000000000000000000000 --- a/gui/.dockerignore +++ /dev/null @@ -1,9 +0,0 @@ -.vscode/ -.git/ -node_modules/ -build/ -public/metainfo/ - -npm-debug.log* -yarn-debug.log* -yarn-error.log* \ No newline at end of file diff --git a/gui/Dockerfile b/gui/Dockerfile deleted file mode 100644 index 810211301dc247e3adc6c22fb3988b5b04a72812..0000000000000000000000000000000000000000 --- a/gui/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2018 Markus Scheidgen -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an"AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This docker image is used for the GUI container that serves the static GUI -# files. -# -# This is a multistage build, one stage for building the GUI and one final stage -# intended for the actual GUI container that serves the GUI. - -# build environment -FROM node:latest as build -RUN mkdir -p /nomad/app -WORKDIR /nomad/app -ENV PATH /nomad/app/node_modules/.bin:$PATH -COPY package.json /nomad/app/package.json -COPY yarn.lock /nomad/app/yarn.lock -RUN yarn -COPY . /nomad/app - -RUN yarn run build - -# production environment -FROM nginx:1.13.9-alpine -COPY --from=build /nomad/app/build /app/nomad - -WORKDIR /app/nomad -COPY ./run.sh run.sh -RUN chmod a+rx run.sh -CMD nginx -g "daemon off;" - -VOLUME /etc/nginx/conf.d - -EXPOSE 8080/tcp diff --git a/gui/run.sh b/gui/run.sh deleted file mode 100644 index 9ee2d2ecf5bf30235b648b336ecd464345954aeb..0000000000000000000000000000000000000000 --- a/gui/run.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -find . -type f -exec sed -i "s_/fairdi/nomad/latest/gui_$1/gui_g" {} \; -nginx -g "daemon off;" \ No newline at end of file diff --git a/gui/version.sh b/gui/version.sh deleted file mode 100755 index b463b6366c891b749d15e10cf7f5d9686544b6b4..0000000000000000000000000000000000000000 --- a/gui/version.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -commit=`git rev-parse --short --verify HEAD` -sed -i -e "s/nomad-gui-commit-placeholder/$commit/g" package.json -rm -f package.json-e diff --git a/nomad/app/__init__.py b/nomad/app/__init__.py index 9959720f7c5dd7d8ea3973e18c30af5ebfb90922..571c3ceb94bc9d615411e1597f07fb3001be9466 100644 --- a/nomad/app/__init__.py +++ b/nomad/app/__init__.py @@ -34,6 +34,7 @@ from nomad import config, utils as nomad_utils from .api import blueprint as api_blueprint, api from .optimade import blueprint as optimade_blueprint, api as optimade from .docs import blueprint as docs_blueprint +from .gui import blueprint as gui_blueprint from . import common @@ -91,6 +92,7 @@ CORS(app) app.register_blueprint(api_blueprint, url_prefix='/api') app.register_blueprint(optimade_blueprint, url_prefix='/optimade') app.register_blueprint(docs_blueprint, url_prefix='/docs') +app.register_blueprint(gui_blueprint, url_prefix='/gui') @app.errorhandler(Exception) diff --git a/nomad/app/api/archive.py b/nomad/app/api/archive.py index 2bbce2321673c3feb995eab27050adacc4854d11..09998b2c6f5f76cb7ccd5d3332ab11ab69110776 100644 --- a/nomad/app/api/archive.py +++ b/nomad/app/api/archive.py @@ -214,7 +214,7 @@ class ArchiveDownloadResource(Resource): _archive_query_model = api.inherit('ArchiveSearch', search_model, { - 'query': fields.Nested(query_model, description='The query used to find the requested entries.'), + 'query': fields.Nested(query_model, description='The query used to find the requested entries.', skip_none=True), 'query_schema': fields.Raw(description='The query schema that defines what archive data to retrive.') }) @@ -225,9 +225,8 @@ class ArchiveQueryResource(Resource): @api.response(400, 'Invalid requests, e.g. wrong owner type or bad search parameters') @api.response(401, 'Not authorized to access the data.') @api.response(404, 'The upload or calculation does not exist') - @api.response(200, 'Archive data send') @api.expect(_archive_query_model) - @api.marshal_with(_archive_query_model, skip_none=True, code=200, description='Search results sent') + @api.marshal_with(_archive_query_model, skip_none=True, code=200, description='Archive search results sent') @authenticate() def post(self): ''' diff --git a/nomad/app/api/common.py b/nomad/app/api/common.py index f69f5570b60d0fec82dabcb802f86d78b4a33367..14661d0d03c02e7bb436da6ac460d7d363d1816e 100644 --- a/nomad/app/api/common.py +++ b/nomad/app/api/common.py @@ -78,7 +78,7 @@ search_model_fields = { search_model = api.model('Search', search_model_fields) query_model_fields = { - qualified_name: fields.Raw(description=quantity.description) + qualified_name: quantity.flask_field for qualified_name, quantity in search.search_quantities.items()} query_model_fields.update(**{ diff --git a/nomad/app/api/dataset.py b/nomad/app/api/dataset.py index a9b8617310ce68591f4395db8dac9577b23b6306..a478d0c72fea226e57ec9ec69da11a7507d99abc 100644 --- a/nomad/app/api/dataset.py +++ b/nomad/app/api/dataset.py @@ -34,7 +34,7 @@ ns = api.namespace( dataset_model = generate_flask_restplus_model(api, Dataset.m_def) dataset_list_model = api.model('DatasetList', { - 'pagination': fields.Nested(model=pagination_model), + 'pagination': fields.Nested(model=pagination_model, skip_none=True), 'results': fields.List(fields.Nested(model=dataset_model, skip_none=True)) }) diff --git a/nomad/app/api/upload.py b/nomad/app/api/upload.py index 625c53e6718d8e54c9ec442389618635d2d3a985..8d883c7fbb4d7f723f6f6bed84b9c5fe59af634b 100644 --- a/nomad/app/api/upload.py +++ b/nomad/app/api/upload.py @@ -80,7 +80,7 @@ upload_model = api.inherit('UploadProcessing', proc_model, { }) upload_list_model = api.model('UploadList', { - 'pagination': fields.Nested(model=pagination_model), + 'pagination': fields.Nested(model=pagination_model, skip_none=True), 'results': fields.List(fields.Nested(model=upload_model, skip_none=True)) }) @@ -103,7 +103,7 @@ upload_with_calcs_model = api.inherit('UploadWithPaginatedCalculations', upload_ 'pagination': fields.Nested(model=api.inherit('UploadCalculationPagination', pagination_model, { 'successes': fields.Integer, 'failures': fields.Integer, - })), + }), skip_none=True), 'results': fields.List(fields.Nested(model=calc_model, skip_none=True)) }), skip_none=True) }) diff --git a/nomad/app/gui.py b/nomad/app/gui.py new file mode 100644 index 0000000000000000000000000000000000000000..e67ac363e02b01c8f1b98e4372c416d81b32e8f4 --- /dev/null +++ b/nomad/app/gui.py @@ -0,0 +1,20 @@ +# Copyright 2018 Markus Scheidgen +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an"AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask import Blueprint +import os.path + +gui_folder = os.path.abspath(os.path.join( + os.path.dirname(__file__), '../../gui/build')) +blueprint = Blueprint('gui', __name__, static_url_path='/', static_folder=gui_folder) diff --git a/nomad/cli/admin/admin.py b/nomad/cli/admin/admin.py index b05776f636c86bb412de30a9e8319c59cb9e9a23..c02cc6231b5121e52705e96fc6538712212ce595 100644 --- a/nomad/cli/admin/admin.py +++ b/nomad/cli/admin/admin.py @@ -229,59 +229,53 @@ def nginx_conf(prefix): server {{ listen 80; server_name www.example.com; + proxy_set_header Host $host; - location /{0} {{ - return 301 /example-nomad/gui; + location / {{ + proxy_pass http://app:8000; }} - location {1}/gui {{ - root /app/; - rewrite ^{1}/gui/(.*)$ /nomad/$1 break; - try_files $uri {1}/gui/index.html; + location ~ {1}\\/?(gui)?$ {{ + rewrite ^ {1}/gui/ permanent; }} - location {1}/gui/service-worker.js {{ - add_header Last-Modified $date_gmt; - add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0'; - if_modified_since off; - expires off; - etag off; - root /app/; - rewrite ^{1}/gui/service-worker.js /nomad/service-worker.js break; + location {1}/gui/ {{ + proxy_intercept_errors on; + error_page 404 = @redirect_to_index; + proxy_pass http://app:8000; + }} + + location @redirect_to_index {{ + rewrite ^ {1}/gui/index.html break; + proxy_pass http://app:8000; }} - location {1}/gui/meta.json {{ + location ~ \\/gui\\/(service-worker\\.js|meta\\.json)$ {{ add_header Last-Modified $date_gmt; add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0'; if_modified_since off; expires off; etag off; - root /app/; - rewrite ^{1}/gui/meta.json /nomad/meta.json break; - }} - - location {1}/api {{ - proxy_set_header Host $host; - proxy_pass_request_headers on; - proxy_pass http://api:8000; + proxy_pass http://app:8000; }} - location {1}/api/uploads {{ + location ~ \\/api\\/uploads\\/?$ {{ client_max_body_size 35g; proxy_request_buffering off; - proxy_set_header Host $host; - proxy_pass_request_headers on; - proxy_pass http://api:8000; + proxy_pass http://app:8000; + }} + + location ~ \\/api\\/(raw|archive) {{ + proxy_buffering off; + proxy_pass http://app:8000; }} - location {1}/api/raw {{ + location ~ \\/api\\/mirror {{ proxy_buffering off; - proxy_set_header Host $host; - proxy_pass_request_headers on; - proxy_pass http://api:8000; + proxy_read_timeout 600; + proxy_pass http://app:8000; }} -}} - '''.format(prefix.lstrip('/'), prefix)) +}}'''.format(prefix)) @ops.command(help=('Generate a proxy pass config for apache2 reverse proxy servers.')) diff --git a/nomad/cli/admin/run.py b/nomad/cli/admin/run.py index 48eab1506d712ab21947e0de8f462adf27dfdb1a..f775ee99c0038458373edfb5fa312d13c1c196e5 100644 --- a/nomad/cli/admin/run.py +++ b/nomad/cli/admin/run.py @@ -43,7 +43,7 @@ def run_app(**kwargs): from nomad import infrastructure from nomad.app.__main__ import run_dev_server infrastructure.setup() - run_dev_server(port=8000, **kwargs) + run_dev_server(port=config.services.api_port, **kwargs) def run_worker(): diff --git a/nomad/cli/client/integrationtests.py b/nomad/cli/client/integrationtests.py index f22dde7a6f5e2af9361c4353c7854a1da7a6f469..c2c0075aaf7dbaf3e10fb8ba9a0c148593bf3133 100644 --- a/nomad/cli/client/integrationtests.py +++ b/nomad/cli/client/integrationtests.py @@ -123,11 +123,22 @@ def integrationtests(ctx, skip_parsers, skip_publish, skip_doi, skip_mirror): assert len(search.results) <= search.pagination.total print('performing archive paginated search') - result = client.archive.archive_query(page=1, per_page=10, **query).response().result + result = client.archive.post_archive_query(payload={ + 'pagination': { + 'page': 1, + 'per_page': 10 + }, + 'query': query + }).response().result assert len(result.results) > 0 print('performing archive scrolled search') - result = client.archive.archive_query(scroll=True, **query).response().result + result = client.archive.post_archive_query(payload={ + 'scroll': { + 'scroll': True + }, + 'query': query + }).response().result assert len(result.results) > 0 print('performing download') diff --git a/nomad/infrastructure.py b/nomad/infrastructure.py index ca54c78b0ace5c5a0b800e064a638fa6bce2d68b..c7c73653d6772372c06613936a899b3f219cb9d5 100644 --- a/nomad/infrastructure.py +++ b/nomad/infrastructure.py @@ -106,8 +106,6 @@ def setup_elastic(): try: from nomad.search import entry_document entry_document.init(index=config.elastic.index_name) - entry_document._index._name = config.elastic.index_name - logger.info('initialized elastic index', index_name=config.elastic.index_name) except RequestError as e: if e.status_code == 400 and 'resource_already_exists_exception' in e.error: # happens if two services try this at the same time @@ -115,6 +113,9 @@ def setup_elastic(): else: raise e + entry_document._index._name = config.elastic.index_name + logger.info('initialized elastic index', index_name=config.elastic.index_name) + return elastic_client diff --git a/nomad/metainfo/search_extension.py b/nomad/metainfo/search_extension.py index 18a7a7d69afb53a55b6d4e3516d8821dd2c716e4..2d83ac00a0cb8c5588694ee313fc1b11183d1059 100644 --- a/nomad/metainfo/search_extension.py +++ b/nomad/metainfo/search_extension.py @@ -166,3 +166,15 @@ class Search(Elastic): @property def many(self): return self.many_and is not None or self.many_or is not None + + @property + def flask_field(self): + from flask_restplus import fields + value_field = fields.String + if self.definition.type == int: + value_field = fields.Integer + + if self.many: + return fields.List(value_field(), description=self.description) + else: + return value_field(description=self.description) diff --git a/ops/containers/README.md b/ops/containers/README.md index 431da73405e647d4540ea8cd7dd0c88bda26d914..611c812252735a45425c191e05925bbdc7d9a2a9 100644 --- a/ops/containers/README.md +++ b/ops/containers/README.md @@ -27,4 +27,16 @@ Changes - added kibana.yml::server.basePath="/nomad/kibana" The file `elk/kibana_objects.json` contains an export of nomad specific searches, -visualizations, and dashboard. \ No newline at end of file +visualizations, and dashboard. + +### CI runner (optional) + +This is the immage that this project uses for its gitlab-ci runner. To build an +push it, you have to log into the project's registry (see [gitlab docs](https://docs.gitlab.com/ee/user/packages/container_registry/)) and do +``` +cd ci-runner +docker build -t gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner . +docker push gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner +``` + +This image allows to bash, git, docker, docker-compose, k8s, and helm3. \ No newline at end of file diff --git a/ops/containers/ci-runner/Dockerfile b/ops/containers/ci-runner/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..911f2d90887a20d2f5c12ba47e5dbbc399c4c421 --- /dev/null +++ b/ops/containers/ci-runner/Dockerfile @@ -0,0 +1,29 @@ +FROM docker/compose +RUN apk update && apk add --no-cache git openssh gettext + +ARG VCS_REF +ARG BUILD_DATE + +# Metadata +LABEL org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.name="helm-kubectl" \ + org.label-schema.url="https://hub.docker.com/r/dtzar/helm-kubectl/" \ + org.label-schema.vcs-url="https://github.com/dtzar/helm-kubectl" \ + org.label-schema.build-date=$BUILD_DATE + +# Note: Latest version of kubectl may be found at: +# https://github.com/kubernetes/kubernetes/releases +ENV KUBE_LATEST_VERSION="v1.18.0" +# Note: Latest version of helm may be found at: +# https://github.com/kubernetes/helm/releases +ENV HELM_VERSION="v3.0.3" + +RUN apk add --no-cache ca-certificates bash git openssh curl \ + && wget -q https://storage.googleapis.com/kubernetes-release/release/${KUBE_LATEST_VERSION}/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl \ + && chmod +x /usr/local/bin/kubectl \ + && wget -q https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz -O - | tar -xzO linux-amd64/helm > /usr/local/bin/helm \ + && chmod +x /usr/local/bin/helm + +WORKDIR /config + +CMD bash \ No newline at end of file diff --git a/ops/docker-compose/nomad/.env b/ops/docker-compose/infrastructure/.env similarity index 100% rename from ops/docker-compose/nomad/.env rename to ops/docker-compose/infrastructure/.env diff --git a/ops/docker-compose/infrastructure/README.md b/ops/docker-compose/infrastructure/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d00341b5c29012f61baaa6d28ab74c2ae6d444f6 --- /dev/null +++ b/ops/docker-compose/infrastructure/README.md @@ -0,0 +1,26 @@ +## Run (dev) infrastructure with docker compose + +You can all necessary databases and other infrastructure with [docker-compose](https://docs.docker.com/compose/) +on a single node/computer that supports docker and docker-compse. + +### How we use docker-compose + +You can use docker-compose to run all necessary databases with one single docker-compose configuration. + The `docker-compose.yml` defines all the different container. + +We use docker-compose overrides to extend a base configuration for different scenarios. +Example docker-compose usage: + +``` +docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d mongo rabbitmq elastic +``` + +The different overrides are: +- *.prod.yml, production (to run the necessary databases for kubenetes deployments) +- *.override.yml, development (development configuration, will be automatically used by docker-compose) +- *.develk.yml, like development but also runs ELK + +To run nomad on top use the nomad command: +``` +nomad admin run appworker +``` \ No newline at end of file diff --git a/ops/docker-compose/nomad/docker-compose.develk.yml b/ops/docker-compose/infrastructure/docker-compose.develk.yml similarity index 65% rename from ops/docker-compose/nomad/docker-compose.develk.yml rename to ops/docker-compose/infrastructure/docker-compose.develk.yml index 93a012c362ec7937bfec59ae30dccb9b86cd4f41..0467d44573b0720c1123142f504e3281ad6ca33e 100644 --- a/ops/docker-compose/nomad/docker-compose.develk.yml +++ b/ops/docker-compose/infrastructure/docker-compose.develk.yml @@ -44,36 +44,3 @@ services: - 5601:5601 # kibana web - 15000:5000 # allow nomad services to logstash outside of docker - 29200:9200 # allows metricbeat config to access es - - # nomad processing worker - worker: - restart: 'no' - build: ../../../ - image: nomad/backend - environment: - NOMAD_LOGSTASH_LEVEL: DEBUG - NOMAD_CONSOLE_LOG_LEVEL: INFO - links: - - elk - - # nomad app - app: - restart: 'no' - image: nomad/backend - environment: - NOMAD_LOGSTASH_LEVEL: DEBUG - NOMAD_CONSOLE_LOG_LEVEL: INFO - depends_on: - - worker - ports: - - 8000:8000 - links: - - elk - - # nomad gui - gui: - restart: 'no' - build: ../../../gui - image: nomad/frontend - ports: - - 8080:8000 diff --git a/ops/docker-compose/nomad/docker-compose.override.yml b/ops/docker-compose/infrastructure/docker-compose.override.yml similarity index 70% rename from ops/docker-compose/nomad/docker-compose.override.yml rename to ops/docker-compose/infrastructure/docker-compose.override.yml index 29b6e55a3873e9594d62ea3cc1cc0e49504bf0b6..cb6b7a753f5a67790e7ff6d2ba439aa0b164f25e 100644 --- a/ops/docker-compose/nomad/docker-compose.override.yml +++ b/ops/docker-compose/infrastructure/docker-compose.override.yml @@ -37,26 +37,3 @@ services: restart: 'no' ports: - 27017:27017 - - # nomad processing worker - worker: - restart: 'no' - build: ../../../ - image: nomad/backend - - # nomad app - app: - restart: 'no' - image: nomad/backend - depends_on: - - worker - ports: - - 8000:8000 - - # nomad gui - gui: - restart: 'no' - build: ../../../gui - image: nomad/frontend - ports: - - 8080:8000 diff --git a/ops/docker-compose/nomad/docker-compose.prod.yml b/ops/docker-compose/infrastructure/docker-compose.prod.yml similarity index 81% rename from ops/docker-compose/nomad/docker-compose.prod.yml rename to ops/docker-compose/infrastructure/docker-compose.prod.yml index fdd010e78edff06a1038c7829882a3158af2b976..cdbe63021c6a276e29c606305c03cd8b3c178b8e 100644 --- a/ops/docker-compose/nomad/docker-compose.prod.yml +++ b/ops/docker-compose/infrastructure/docker-compose.prod.yml @@ -49,23 +49,3 @@ services: - 5601:5601 # kibana web - 5000:5000 - 9201:9200 # allows metricbeat config to access es - - app: - environment: - NOMAD_LOGSTASH_LEVEL: INFO - links: - - elk - volumes: - - /nomad:/nomad - - worker: - environment: - NOMAD_LOGSTASH_LEVEL: INFO - links: - - elk - volumes: - - /nomad:/nomad - - gui: - ports: - - 80:80 diff --git a/ops/docker-compose/nomad/docker-compose.yml b/ops/docker-compose/infrastructure/docker-compose.yml similarity index 61% rename from ops/docker-compose/nomad/docker-compose.yml rename to ops/docker-compose/infrastructure/docker-compose.yml index 1f936af4e99d352cb444a084d3217939af8763a7..35b45bf1e649f2299774b0a7bc38599650487d43 100644 --- a/ops/docker-compose/nomad/docker-compose.yml +++ b/ops/docker-compose/infrastructure/docker-compose.yml @@ -66,52 +66,8 @@ services: - nomad_mongo:/data/db command: mongod --logpath=/dev/null # --quiet - # nomad processing worker - worker: - restart: always - image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest - container_name: nomad_worker - environment: - <<: *nomad_backend_env - NOMAD_SERVICE: nomad_worker - links: - - keycloak - - rabbitmq - - elastic - - mongo - volumes: - - nomad_files:/app/.volumes/fs - command: python -m celery worker -l info -A nomad.processing -Q celery,cacls,uploads - - # nomad app - app: - restart: always - image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest - container_name: nomad_api - environment: - <<: *nomad_backend_env - NOMAD_SERVICE: nomad_api - links: - - keycloak - - rabbitmq - - elastic - - mongo - volumes: - - nomad_files:/app/.volumes/fs - command: python -m gunicorn.app.wsgiapp -w 4 --log-config ops/gunicorn.log.conf -b 0.0.0.0:8000 --timeout 300 nomad.app:app - - # nomad gui - gui: - restart: always - image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest - container_name: nomad_gui - command: nginx -g 'daemon off;' - links: - - app - volumes: nomad_keycloak: nomad_mongo: nomad_elastic: nomad_rabbitmq: - nomad_files: diff --git a/ops/docker-compose/nomad-oasis/README.md b/ops/docker-compose/nomad-oasis/README.md index b4a9a2375c4e2588a78a010bc68f2787866d795a..98249a90e0f57f2df4493129de1605cc4e9860e9 100644 --- a/ops/docker-compose/nomad-oasis/README.md +++ b/ops/docker-compose/nomad-oasis/README.md @@ -104,7 +104,7 @@ services: # nomad worker (processing) worker: restart: always - image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable + image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest container_name: nomad_oasis_worker environment: <<: *nomad_backend_env @@ -118,10 +118,10 @@ services: - ./nomad.yaml:/app/nomad.yaml command: python -m celery worker -l info -A nomad.processing -Q celery,calcs,uploads - # nomad app (api) + # nomad app (api + gui) app: restart: always - image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable + image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest container_name: nomad_oasis_app environment: <<: *nomad_backend_env @@ -133,22 +133,23 @@ services: volumes: - nomad_oasis_files:/app/.volumes/fs - ./nomad.yaml:/app/nomad.yaml - command: python -m gunicorn.app.wsgiapp -w 4 --log-config ops/gunicorn.log.conf -b 0.0.0.0:8000 --timeout 300 nomad.app:app + - ./env.js:/app/gui/build/env.js + - ./gunicorn.log.conf:/app/gunicorn.log.conf + - ./gunicorn.conf:/app/gunicorn.conf + command: ["./run.sh", "/nomad-oasis"] - # nomad gui (serving the js) + # nomad gui (a reverse proxy for nomad) gui: restart: always - image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:stable + image: nginx:1.13.9-alpine container_name: nomad_oasis_gui command: nginx -g 'daemon off;' volumes: - ./nginx.conf:/etc/nginx/conf.d/default.conf - - ./env.js:/app/nomad/env.js links: - app ports: - 80:80 - command: ["./run.sh", "/example-nomad"] volumes: nomad_oasis_mongo: @@ -161,7 +162,7 @@ There are no mandatory changes necessary. A few things to notice: - All services use docker volumes for storage. This could be changed to host mounts. -- It mounts three configuration files that need to be provided (see below): `nomad.yaml`, `nginx.conf`, `env.js`. +- It mounts three configuration files that need to be provided (see below): `nomad.yaml`, `nginx.conf`, `env.js`, `gunicorn.conf`, `gunicorn.log.conf`. - The only exposed port is `80`. This could be changed to a desired port if necessary. - The NOMAD images are pulled from our gitlab in Garching, the other services use images from a public registry (*dockerhub*). - All container will be named `nomad_oasis_*`. These names can be used to later reference the container with the `docker` cmd. @@ -214,13 +215,13 @@ You need to change: ### nginx.conf -The GUI container also serves as a proxy that forwards request to the app container. The +The GUI container serves as a proxy that forwards request to the app container. The proxy is an nginx server and needs a configuration similar to this: ``` server { listen 80; - server_name <your-host>; + server_name www.example.com; location /nomad-oasis { proxy_set_header Host $host; @@ -228,20 +229,13 @@ server { proxy_pass http://app:8000; } - location /nomad-oasis/gui { - root /app/; - rewrite ^/nomad-oasis/gui/(.*)$ /nomad/$1 break; - try_files $uri /nomad-oasis/gui/index.html; - } - location /nomad-oasis/gui/service-worker.js { add_header Last-Modified $date_gmt; add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0'; if_modified_since off; expires off; etag off; - root /app/; - rewrite ^/nomad-oasis/gui/service-worker.js /nomad/service-worker.js break; + proxy_pass http://app:8000; } location /nomad-oasis/api/uploads { @@ -270,6 +264,12 @@ A few things to notice: - `client_max_body_size` sets a limit to the possible upload size. - If you operate the GUI container behind another proxy, keep in mind that your proxy should not buffer requests/responses to allow streaming of large requests/responses for `../api/uploads` and `../api/raw`. +### gunicorn + +Simply create empty `gunicorn.conf` and `gunicorn.log.conf` in the beginning. Gunicorn +is the WSGI-server that runs the nomad app. Consult the [gunicorn documentation](https://docs.gunicorn.org/en/stable/configure.html) +for configuration options. + ## Starting and stopping If you prepared the above files, simply use the usual `docker-compose` commands to start everything. diff --git a/ops/docker-compose/nomad-oasis/docker-compose.yml b/ops/docker-compose/nomad-oasis/docker-compose.yml index e78cb64f3f10ad465a80cd9ddd0f3df39b876084..a9223411af06a3cb9c0cb63a315bc1c14f8b06ab 100644 --- a/ops/docker-compose/nomad-oasis/docker-compose.yml +++ b/ops/docker-compose/nomad-oasis/docker-compose.yml @@ -56,7 +56,7 @@ services: # nomad worker (processing) worker: restart: always - image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest + image: nomad/local:latest container_name: nomad_oasis_worker environment: <<: *nomad_backend_env @@ -70,10 +70,10 @@ services: - ./nomad.yaml:/app/nomad.yaml command: python -m celery worker -l info -A nomad.processing -Q celery,calcs,uploads - # nomad app (api) + # nomad app (api + gui) app: restart: always - image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest + image: nomad/local:latest container_name: nomad_oasis_app environment: <<: *nomad_backend_env @@ -85,22 +85,23 @@ services: volumes: - nomad_oasis_files:/app/.volumes/fs - ./nomad.yaml:/app/nomad.yaml - command: python -m gunicorn.app.wsgiapp -w 4 --log-config ops/gunicorn.log.conf -b 0.0.0.0:8000 --timeout 300 nomad.app:app + - ./env.js:/app/gui/build/env.js + - ./gunicorn.log.conf:/app/gunicorn.log.conf + - ./gunicorn.conf:/app/gunicorn.conf + command: ["./run.sh", "/nomad-oasis"] - # nomad gui (serving the js) + # nomad gui (a reverse proxy for nomad) gui: restart: always - image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest + image: nginx:1.13.9-alpine container_name: nomad_oasis_gui command: nginx -g 'daemon off;' volumes: - ./nginx.conf:/etc/nginx/conf.d/default.conf - - ./env.js:/app/nomad/env.js links: - app ports: - 80:80 - command: ["./run.sh", "/nomad-oasis"] volumes: nomad_oasis_mongo: diff --git a/ops/docker-compose/nomad-oasis/gunicorn.conf b/ops/docker-compose/nomad-oasis/gunicorn.conf new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/ops/gunicorn.log.conf b/ops/docker-compose/nomad-oasis/gunicorn.log.conf similarity index 100% rename from ops/gunicorn.log.conf rename to ops/docker-compose/nomad-oasis/gunicorn.log.conf diff --git a/ops/docker-compose/nomad-oasis/nginx.conf b/ops/docker-compose/nomad-oasis/nginx.conf index e13f66a001473c6d600b747196aee77ccc4342f3..8ee7f3884f6611469714e37aa437b662d7f8f8d8 100644 --- a/ops/docker-compose/nomad-oasis/nginx.conf +++ b/ops/docker-compose/nomad-oasis/nginx.conf @@ -1,41 +1,50 @@ server { listen 80; server_name www.example.com; + proxy_set_header Host $host; - location /nomad-oasis { - proxy_set_header Host $host; - proxy_pass_request_headers on; + location / { proxy_pass http://app:8000; } - location /nomad-oasis/gui { - root /app/; - rewrite ^/nomad-oasis/gui/(.*)$ /nomad/$1 break; - try_files $uri /nomad-oasis/gui/index.html; + location ~ /nomad-oasis\/?(gui)?$ { + rewrite ^ /nomad-oasis/gui/ permanent; } - location /nomad-oasis/gui/service-worker.js { + location /nomad-oasis/gui/ { + proxy_intercept_errors on; + error_page 404 = @redirect_to_index; + proxy_pass http://app:8000; + } + + location @redirect_to_index { + rewrite ^ /nomad-oasis/gui/index.html break; + proxy_pass http://app:8000; + } + + location ~ \/gui\/(service-worker\.js|meta\.json)$ { add_header Last-Modified $date_gmt; add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0'; if_modified_since off; expires off; etag off; - root /app/; - rewrite ^/nomad-oasis/gui/service-worker.js /nomad/service-worker.js break; + proxy_pass http://app:8000; } - location /nomad-oasis/api/uploads { + location ~ \/api\/uploads\/?$ { client_max_body_size 35g; proxy_request_buffering off; - proxy_set_header Host $host; - proxy_pass_request_headers on; proxy_pass http://app:8000; } - location /nomad-oasis/api/raw { + location ~ \/api\/(raw|archive) { + proxy_buffering off; + proxy_pass http://app:8000; + } + + location ~ \/api\/mirror { proxy_buffering off; - proxy_set_header Host $host; - proxy_pass_request_headers on; + proxy_read_timeout 600; proxy_pass http://app:8000; } } \ No newline at end of file diff --git a/ops/docker-compose/nomad/README.md b/ops/docker-compose/nomad/README.md deleted file mode 100644 index 9e9eeb8bbc08f65dca2cda1c9a60e17b1277dd17..0000000000000000000000000000000000000000 --- a/ops/docker-compose/nomad/README.md +++ /dev/null @@ -1,47 +0,0 @@ -## Single Node NOMAD Deployment, Using Docker Compose - -You can run NOMAD with [docker-compose](https://docs.docker.com/compose/) on a single node -that supports docker and docker-compse. The docker-compose files are part of the -[NOMAD source code](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR) -and can be found under `ops/docker-compose/nomad`. - -### How we use docker-compose - -You can use docker-compose to run all necessary databases, the API, the worker, and -gui with one single docker-compose configuration. The `docker-compose.yml` defines -all the different container. - -We use docker-compose overrides to extend a base configuration for different scenarios. -Example docker-compose usage: - -``` -docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d app -``` - -The different overrides are: -- *.prod.yml, production (to run the necessary databases for kubenetes deployments) -- *.override.yml, development (development configuration, will be automatically used by docker-compose) -- *.develk.yml, like development but also runs ELK - -Within the overrides you can configure the NOMAD containers (app, worker, gui). - -### Configuring the API and worker - -The API and worker can be configured through a nomad.yaml file or with environment -variables. A nomad.yaml files needs to be mounted to `/app/nomad.yaml`. There are -[several options](https://docs.docker.com/compose/environment-variables/) to set -environment variables in docker-compose. - -### Configuring the GUI - -This encompasses to parts a `env.js` file for the client side GUI code. And an -`nginx.conf` for the web server running the GUI (and reverse proxying the API). - -Both need to be mounted to the respective containers. The `env.js` under `/app/nomad/env.js` -and the `nginx.conf` under `/etc/nginx/conf.d/default.conf`. Usually the only configuration -item for the `nginx.conf` is the desired URL path prefix. You can generate an `nginx.conf` -with: - -``` -nomad admin nginx-conf --prefix /example-nomad -``` diff --git a/ops/helm/nomad/ci-dev-values.yaml b/ops/helm/nomad/ci-dev-values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69654013ef508aa251b0bf70295ec3f50469632a --- /dev/null +++ b/ops/helm/nomad/ci-dev-values.yaml @@ -0,0 +1,41 @@ +image: + pullPolicy: "Always" + +proxy: + external: + host: "labdev-nomad.esc.rzg.mpg.de" + +ingress: + enabled: true + +gui: + debug: true + +worker: + replicas: 1 + routing: "queue" + +elastic: + host: elastic + +mongo: + host: mongo + +logstash: + host: logstash + +dbname: nomad_dev_v0_8 + +keycloak: + serverUrl: "https://repository.nomad-coe.eu/fairdi/keycloak/auth/" + passwordSecret: 'nomad-keycloak-password' + realmName: 'fairdi_nomad_prod' + clientId: 'nomad_public' + admin_user_id: '82efac55-6187-408c-8027-b98580c0e1c5' + +volumes: + prefixSize: 1 + public: /nomad/fairdi/dev/v0_8/fs/public + staging: /nomad/fairdi/dev/v0_8/fs/staging + tmp: /nomad/fairdi/dev/v0_8/fs/tmp + nomad: /nomad diff --git a/ops/helm/nomad/requirements.lock b/ops/helm/nomad/requirements.lock index c98ee662807a34ff2f8a21b01efe7c896afa2954..acf87a452c409be23d65ee3064eba5893f60a878 100644 --- a/ops/helm/nomad/requirements.lock +++ b/ops/helm/nomad/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: rabbitmq - repository: https://kubernetes-charts.storage.googleapis.com/ - version: 5.1.0 -digest: sha256:c93d68af945507f31e38cb7badd0781893e5541a93f95855157d7016dbf5ef6f -generated: 2019-08-28T16:37:38.419893741+02:00 + repository: https://charts.bitnami.com/bitnami + version: 6.21.0 +digest: sha256:71daa0c62bb26ed91bb50b8136f50e29d45b72510b668623f6713b4e52faa206 +generated: "2020-03-30T12:08:12.998147583+02:00" diff --git a/ops/helm/nomad/requirements.yaml b/ops/helm/nomad/requirements.yaml index 78032e8fd3de2337af70259caf6b2421ebe271c3..5a1951dda5c116d2788198d6ccc7b5f9b12f8013 100644 --- a/ops/helm/nomad/requirements.yaml +++ b/ops/helm/nomad/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: rabbitmq - version: "5.1.0" - repository: "https://kubernetes-charts.storage.googleapis.com/" + version: "6.21.0" + repository: "https://charts.bitnami.com/bitnami" diff --git a/ops/helm/nomad/templates/api-deployment.yaml b/ops/helm/nomad/templates/api-deployment.yaml index 2c1a589710ac8ef647b5e0a060934cf7939bdbb7..156728abbc516d91ca46b697af2a9a591c029deb 100644 --- a/ops/helm/nomad/templates/api-deployment.yaml +++ b/ops/helm/nomad/templates/api-deployment.yaml @@ -74,6 +74,28 @@ data: worker_connections = 1000 workers = {{ .Values.app.worker }} --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nomad.fullname" . }}-gui-env-js + labels: + app.kubernetes.io/name: {{ include "nomad.name" . }}-gui-env-js + helm.sh/chart: {{ include "nomad.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + env.js: | + window.nomadEnv = { + "appBase": "{{ .Values.proxy.external.path }}", + "keycloakBase": "{{ .Values.keycloak.serverExternalUrl }}", + "keycloakRealm": "{{ .Values.keycloak.realmName }}", + "keycloakClientId": "{{ .Values.keycloak.guiClientId }}", + "matomoSiteId": {{ .Values.gui.matomoSiteId }}, + "matomoUrl": "{{ .Values.gui.matomoUrl }}", + "sendTrackingData": {{ .Values.gui.sendTrackingData }}, + "debug": {{ .Values.gui.debug }} + }; +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -94,12 +116,15 @@ spec: labels: app.kubernetes.io/name: {{ include "nomad.name" . }}-app app.kubernetes.io/instance: {{ .Release.Name }} - date: "{{ .Release.Time.Seconds }}" + {{ if .Values.roll }} + annotations: + rollme: {{ randAlphaNum 5 | quote }} + {{ end }} spec: containers: - name: {{ include "nomad.name" . }}-app - image: "{{ .Values.images.nomad.name }}:{{ .Values.images.nomad.tag }}" - imagePullPolicy: Always + image: "{{ .Values.image.name }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} volumeMounts: - mountPath: /app/nomad.yaml name: nomad-conf @@ -116,6 +141,10 @@ spec: name: staging-volume - mountPath: /nomad name: nomad-volume + - mountPath: /app/gui/build/env.js + readOnly: true + subPath: env.js + name: gui-env-js env: - name: NOMAD_SERVICE value: "app" @@ -163,7 +192,7 @@ spec: name: {{ .Values.datacite.secret }} key: user {{ end }} - command: ["python", "-m", "gunicorn.app.wsgiapp", "--config", "gunicorn.conf", "--log-config", "gunicorn.log.conf", "-b 0.0.0.0:8000", "nomad.app:app"] + command: ["./run.sh", "{{ .Values.proxy.external.path }}"] livenessProbe: httpGet: path: "{{ .Values.proxy.external.path }}/alive" @@ -181,7 +210,7 @@ spec: nodeSelector: nomadtype: {{ .Values.app.nomadNodeType }} imagePullSecrets: - - name: {{ .Values.images.secret }} + - name: {{ .Values.image.secret }} volumes: - name: gunicorn-log-conf configMap: @@ -192,6 +221,9 @@ spec: - name: nomad-conf configMap: name: {{ include "nomad.fullname" . }}-configmap + - name: gui-env-js + configMap: + name: {{ include "nomad.fullname" . }}-gui-env-js - name: public-volume hostPath: path: {{ .Values.volumes.public }} diff --git a/ops/helm/nomad/templates/gui-deployment.yml b/ops/helm/nomad/templates/gui-deployment.yml index 968c7a9c4abe503697efdfc86fa4df8160bdad89..3a844a64b5d0e59d3ca2df8ce67669033b369dbe 100644 --- a/ops/helm/nomad/templates/gui-deployment.yml +++ b/ops/helm/nomad/templates/gui-deployment.yml @@ -12,80 +12,56 @@ data: server { listen 80; server_name www.example.com; + proxy_set_header Host $host; - location {{ .Values.proxy.external.path }} { - proxy_set_header Host $host; - proxy_pass_request_headers on; + proxy_connect_timeout {{ .Values.proxy.timeout }}; + proxy_read_timeout {{ .Values.proxy.timeout }}; + + location / { proxy_pass http://{{ include "nomad.fullname" . }}-app:8000; - proxy_connect_timeout {{ .Values.proxy.timeout }}; - proxy_read_timeout {{ .Values.proxy.timeout }}; } - location {{ .Values.proxy.external.path }}/gui { - root /app/; - rewrite ^{{ .Values.proxy.external.path }}/gui/(.*)$ /nomad/$1 break; - try_files $uri {{ .Values.proxy.external.path }}/gui/index.html; + location ~ {{ .Values.proxy.external.path }}\/?(gui)?$ { + rewrite ^ {{ .Values.proxy.external.path }}/gui/ permanent; } - location {{ .Values.proxy.external.path }}/gui/service-worker.js { - add_header Last-Modified $date_gmt; - add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0'; - if_modified_since off; - expires off; - etag off; - root /app/; - rewrite ^{{ .Values.proxy.external.path }}/gui/service-worker.js /nomad/service-worker.js break; + location {{ .Values.proxy.external.path }}/gui/ { + proxy_intercept_errors on; + error_page 404 = @redirect_to_index; + proxy_pass http://{{ include "nomad.fullname" . }}-app:8000; } - location {{ .Values.proxy.external.path }}/gui/meta.json { + location @redirect_to_index { + rewrite ^ {{ .Values.proxy.external.path }}/gui/index.html break; + proxy_pass http://{{ include "nomad.fullname" . }}-app:8000; + } + + location ~ \/gui\/(service-worker\.js|meta\.json)$ { add_header Last-Modified $date_gmt; add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0'; if_modified_since off; expires off; etag off; - root /app/; - rewrite ^{{ .Values.proxy.external.path }}/gui/meta.json /nomad/meta.json break; + proxy_pass http://{{ include "nomad.fullname" . }}-app:8000; } - location {{ .Values.proxy.external.path }}/api/uploads { + location ~ \/api\/uploads\/?$ { client_max_body_size 35g; proxy_request_buffering off; - proxy_set_header Host $host; - proxy_pass_request_headers on; proxy_pass http://{{ include "nomad.fullname" . }}-app:8000; - proxy_connect_timeout {{ .Values.proxy.timeout }}; - proxy_send_timeout {{ .Values.proxy.timeout }}; } - location {{ .Values.proxy.external.path }}/api/raw { + location ~ \/api\/(raw|archive) { proxy_buffering off; - proxy_set_header Host $host; - proxy_pass_request_headers on; proxy_pass http://{{ include "nomad.fullname" . }}-app:8000; - proxy_connect_timeout {{ .Values.proxy.timeout }}; - proxy_read_timeout {{ .Values.proxy.timeout }}; } - location {{ .Values.proxy.external.path }}/api/mirror { + location ~ \/api\/mirror { proxy_buffering off; - proxy_set_header Host $host; - proxy_pass_request_headers on; + proxy_read_timeout {{ .Values.proxy.mirrorTimeout }}; proxy_pass http://{{ include "nomad.fullname" . }}-app:8000; - proxy_connect_timeout {{ .Values.proxy.timeout }}; - proxy_read_timeout {{ .Values.proxy.timeout }}; } } - env.js: | - window.nomadEnv = { - "appBase": "{{ .Values.proxy.external.path }}", - "keycloakBase": "{{ .Values.keycloak.serverExternalUrl }}", - "keycloakRealm": "{{ .Values.keycloak.realmName }}", - "keycloakClientId": "{{ .Values.keycloak.guiClientId }}", - "matomoSiteId": {{ .Values.gui.matomoSiteId }}, - "matomoUrl": "{{ .Values.gui.matomoUrl }}", - "sendTrackingData": {{ .Values.gui.sendTrackingData }}, - "debug": {{ .Values.gui.debug }} - }; --- apiVersion: apps/v1 kind: Deployment @@ -107,22 +83,21 @@ spec: labels: app.kubernetes.io/name: {{ include "nomad.name" . }}-gui app.kubernetes.io/instance: {{ .Release.Name }} + {{ if .Values.roll }} + annotations: + rollme: {{ randAlphaNum 5 | quote }} + {{ end }} spec: containers: - name: {{ include "nomad.name" . }}-gui - image: "{{ .Values.images.frontend.name }}:{{ .Values.images.frontend.tag }}" - imagePullPolicy: Always - command: ["./run.sh", "{{ .Values.proxy.external.path }}"] + image: "nginx:1.13.9-alpine" + command: ["nginx", "-g", "daemon off;"] ports: - containerPort: 80 volumeMounts: - mountPath: /etc/nginx/conf.d readOnly: true name: nginx-conf - - mountPath: /app/nomad/env.js - readOnly: true - subPath: env.js - name: nomad-app - mountPath: /var/log/nginx name: log livenessProbe: @@ -139,8 +114,6 @@ spec: periodSeconds: 3 nodeSelector: nomadtype: public - imagePullSecrets: - - name: {{ .Values.images.secret }} volumes: - name: nginx-conf configMap: @@ -148,11 +121,5 @@ spec: items: - key: nginx.conf path: default.conf - - name: nomad-app - configMap: - name: {{ include "nomad.fullname" . }}-gui-config - items: - - key: env.js - path: env.js - name: log emptyDir: {} diff --git a/ops/helm/nomad/templates/gui-service.yaml b/ops/helm/nomad/templates/gui-service.yaml index 8466ba0824b40df1cf79f4ffa3fd51d3ce0bd1f4..c2dea28d87bafa9bedc0b8bbdde3657be885c7d1 100644 --- a/ops/helm/nomad/templates/gui-service.yaml +++ b/ops/helm/nomad/templates/gui-service.yaml @@ -8,16 +8,19 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} spec: + {{ if .Values.nodePort }} type: NodePort externalIPs: - {{ .Values.proxy.nodeIP }} + {{ end }} ports: - - nodePort: {{ .Values.proxy.nodePort }} - port: 80 + - port: 80 targetPort: 80 protocol: TCP name: http - + {{ if .Values.nodePort }} + nodePort: {{ .Values.proxy.nodePort }} + {{ end }} selector: app.kubernetes.io/name: {{ include "nomad.name" . }}-gui app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/ops/helm3/nomad/templates/ingress.yaml b/ops/helm/nomad/templates/ingress.yaml similarity index 100% rename from ops/helm3/nomad/templates/ingress.yaml rename to ops/helm/nomad/templates/ingress.yaml diff --git a/ops/helm/nomad/templates/nomad-configmap.yml b/ops/helm/nomad/templates/nomad-configmap.yml index 55490e53eee3dfa1355d965d79f6724399a0d174..3b3388fb6a2bf14b7f8b4b24dd841691127b5475 100644 --- a/ops/helm/nomad/templates/nomad-configmap.yml +++ b/ops/helm/nomad/templates/nomad-configmap.yml @@ -24,7 +24,7 @@ data: api_port: {{ .Values.proxy.external.port }} api_base_path: "{{ .Values.proxy.external.path }}" api_secret: "{{ .Values.api.secret }}" - https: {{ .Values.app.https }} + https: {{ .Values.proxy.external.https }} upload_limit: {{ .Values.api.uploadLimit }} admin_user_id: {{ .Values.keycloak.admin_user_id }} rabbitmq: diff --git a/ops/helm/nomad/templates/worker-deployment.yaml b/ops/helm/nomad/templates/worker-deployment.yaml index 1df5f52ac7fc9859a4528f7d889c8aa14612bb90..26f29849bb4934358d56f607b121abdd6ec68a9e 100644 --- a/ops/helm/nomad/templates/worker-deployment.yaml +++ b/ops/helm/nomad/templates/worker-deployment.yaml @@ -18,7 +18,10 @@ spec: labels: app.kubernetes.io/name: {{ include "nomad.name" . }}-worker app.kubernetes.io/instance: {{ .Release.Name }} - date: "{{ .Release.Time.Seconds }}" + {{ if .Values.roll }} + annotations: + rollme: {{ randAlphaNum 5 | quote }} + {{ end }} spec: affinity: podAntiAffinity: @@ -30,8 +33,8 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} containers: - name: {{ include "nomad.name" . }}-worker - image: "{{ .Values.images.nomad.name }}:{{ .Values.images.nomad.tag }}" - imagePullPolicy: Always + image: "{{ .Values.image.name }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} resources: limits: memory: "{{ .Values.worker.memlimit }}Gi" @@ -94,12 +97,12 @@ spec: - bash - -c - NOMAD_LOGSTASH_LEVEL=WARNING python -m celery -A nomad.processing status | grep "${NOMAD_CELERY_NODE_NAME}:.*OK" - initialDelaySeconds: 5 - periodSeconds: 120 + initialDelaySeconds: 15 + periodSeconds: 30 nodeSelector: nomadtype: {{ .Values.worker.nomadNodeType }} imagePullSecrets: - - name: {{ .Values.images.secret }} + - name: {{ .Values.image.secret }} volumes: - name: nomad-conf configMap: diff --git a/ops/helm/nomad/values.yaml b/ops/helm/nomad/values.yaml index 264d5d56e5b33779b48976ce1fc4ce2354069f26..7ca7a911600e35ee30025060b15ed880ba4a12d1 100644 --- a/ops/helm/nomad/values.yaml +++ b/ops/helm/nomad/values.yaml @@ -1,34 +1,35 @@ ## Default values for nomad@FAIRDI ## Everything concerning the container images to be used -images: +image: ## The kubernetes docker-registry secret that can be used to access the registry # with the container image in it. # It can be created via: # kubectl create secret docker-registry gitlab-mpcdf --docker-server=gitlab-registry.mpcdf.mpg.de --docker-username=<your-user-name > --docker-password=<yourpass> --docker-email=<email> secret: gitlab-mpcdf - ## The nomad image with all nomad relavant python code. Used by api and worker service. - nomad: - ## The docker container image name without tag - name: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair - ## The docker container image tag - tag: latest - pullPolicy: IfNotPresent - - ## The nomad frontend image - frontend: - ## The docker container image name without tag - name: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend - ## The docker container image tag - tag: latest - pullPolicy: IfNotPresent + ## The docker container image name without tag + name: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair + ## The docker container image tag + tag: latest + pullPolicy: IfNotPresent +## Ingress can be unable to provide gui and api access through kubernetes ingress (only k8s 1.18+) +ingress: + enabled: false + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/proxy-body-size: "32g" + nginx.ingress.kubernetes.io/proxy-request-buffering: "off" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "10" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + hosts: + - "" ## Everything concerning the nomad app app: replicas: 1 - https: true ## Number of gunicorn worker. worker: 10 ## Number of threads per gunicorn worker (for async workerClass) @@ -43,19 +44,15 @@ app: api: ## Secret used as cryptographic seed secret: "defaultApiSecret" - ## The adminstrator password (only way to ever set/change it) - adminPassword: "password" - ## A kubernetes secret with administrator password (supersedes adminPassword) - adminPasswordSecret: null ## Limit of unpublished uploads per user, except admin user uploadLimit: 10 ## Everything concerning the nomad worker worker: replicas: 1 - # request and limit in GB - memrequest: 64 - memlimit: 420 + # request and limit in GB, good prod sizes are 64, 420 + memrequest: 8 + memlimit: 32 console_loglevel: ERROR logstash_loglevel: INFO ## There are two routing modes "queue" and "worker". The "queue" routing will use a general @@ -80,14 +77,16 @@ gui: ## Everything concerning the nginx that serves the gui, proxies the api # It is run via NodePort service proxy: - nodePort: 30001 - nodeIP: 130.183.207.104 + # Set a nodePort to create a NodePort service instead of ClusterIP. Also set a nodeIP for the externalIP. + nodePort: + nodeIP: timeout: 120 + mirrorTimeout: 600 external: host: "labdev-nomad.esc.rzg.mpg.de" port: 80 path: "/fairdi/nomad/latest" - kibanaPath: "/fairdi/kibana" + https: true ## configuration of the chart dependency for rabbitmq rabbitmq: @@ -104,11 +103,9 @@ rabbitmq: ## A common name/prefix for all dbs and indices. dbname: fairdi_nomad_latest -## The url for the upload page, used in emails to forward the user -uploadurl: 'https://labdev-nomad.esc.mpg.rzg.de/fairdi/nomad/latest/uploads' - ## Databases that are not run within the cluster. # To run databases in the cluster, use the nomad-full helm chart. + mongo: host: nomad-flink-01.esc port: 27017 @@ -145,7 +142,7 @@ keycloak: ## Everything concerning the data that is used by the service volumes: - prefixSize: 2 + prefixSize: 1 public: /nomad/fairdi/latest/fs/public staging: /nomad/fairdi/latest/fs/staging tmp: /nomad/fairdi/latest/fs/tmp diff --git a/ops/helm3/nomad/.gitignore b/ops/helm3/nomad/.gitignore deleted file mode 100644 index 711a39c541afc04f86b51e268b350d204f48c875..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/.gitignore +++ /dev/null @@ -1 +0,0 @@ -charts/ \ No newline at end of file diff --git a/ops/helm3/nomad/.helmignore b/ops/helm3/nomad/.helmignore deleted file mode 100644 index 50af0317254197a5a019f4ac2f8ecc223f93f5a7..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/.helmignore +++ /dev/null @@ -1,22 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/ops/helm3/nomad/Chart.yaml b/ops/helm3/nomad/Chart.yaml deleted file mode 100644 index ae5c367983977b455a0f4f4ffa06a50be7d9c7f9..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -appVersion: "0.8.0" -description: A Helm chart for Kubernetes that only runs nomad services and uses externally hosted databases. -name: nomad -version: 0.8.0 diff --git a/ops/helm3/nomad/README.md b/ops/helm3/nomad/README.md deleted file mode 100644 index 0b98e975fb46645d12dc1c4b665a0c279114f6fb..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/README.md +++ /dev/null @@ -1,16 +0,0 @@ -## Cluster Deployment, Using Kubernetes and Helm - -We use helm charts to describe the deployment of nomad services in a kubernetes cluster. -The NOMAD chart is part of the -[NOMAD source code](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR) -and can be found under `ops/helm/nomad`. - -This chart allows to run the nomad app, worker, gui, and proxy in a kubernetes cluster. -The `values.yaml` contains more documentation on the different values. - -The chart can be used to run multiple nomad instances in parallel on the same cluster, -by using different URL-path and database names. - -The chart does not run any databases and search engines. Those are supposed to run -separately (see also *nomad-full* for an alternative approach) and their hosts, etc. -can be configures via helm values. diff --git a/ops/helm3/nomad/requirements.lock b/ops/helm3/nomad/requirements.lock deleted file mode 100644 index acf87a452c409be23d65ee3064eba5893f60a878..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/requirements.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: rabbitmq - repository: https://charts.bitnami.com/bitnami - version: 6.21.0 -digest: sha256:71daa0c62bb26ed91bb50b8136f50e29d45b72510b668623f6713b4e52faa206 -generated: "2020-03-30T12:08:12.998147583+02:00" diff --git a/ops/helm3/nomad/requirements.yaml b/ops/helm3/nomad/requirements.yaml deleted file mode 100644 index 5a1951dda5c116d2788198d6ccc7b5f9b12f8013..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/requirements.yaml +++ /dev/null @@ -1,4 +0,0 @@ -dependencies: -- name: rabbitmq - version: "6.21.0" - repository: "https://charts.bitnami.com/bitnami" diff --git a/ops/helm3/nomad/templates/_helpers.tpl b/ops/helm3/nomad/templates/_helpers.tpl deleted file mode 100644 index 1d5277e45c79510691d3311e97bea01702d3f5bd..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/templates/_helpers.tpl +++ /dev/null @@ -1,32 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "nomad.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "nomad.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "nomad.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/ops/helm3/nomad/templates/api-deployment.yaml b/ops/helm3/nomad/templates/api-deployment.yaml deleted file mode 100644 index b49b0cfe307d482cd634ed2c95165b861a5067f1..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/templates/api-deployment.yaml +++ /dev/null @@ -1,214 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "nomad.fullname" . }}-app-gunicorn-log-config - labels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-app-gunicorn-log-config - helm.sh/chart: {{ include "nomad.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -data: - gunicorn.log.conf: | - [loggers] - keys=root, gunicorn.error, gunicorn.access - - [handlers] - keys=console, access, error - - [formatters] - keys=generic, json - - [logger_root] - level=INFO - handlers=console - - [logger_gunicorn.error] - level=INFO - handlers=error - qualname=gunicorn.error - - [logger_gunicorn.access] - level=INFO - handlers=access - qualname=gunicorn.access - - [handler_console] - class=StreamHandler - formatter=generic - args=(sys.stdout, ) - - [handler_access] - class=StreamHandler - formatter=json - args=(sys.stdout, ) - - [handler_error] - class=StreamHandler - formatter=json - args=(sys.stdout, ) - - [formatter_generic] - format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s - datefmt=%Y-%m-%d %H:%M:%S - class=logging.Formatter - - [formatter_json] - class=pythonjsonlogger.jsonlogger.JsonFormatter ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "nomad.fullname" . }}-app-gunicorn-config - labels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-app-gunicorn-config - helm.sh/chart: {{ include "nomad.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -data: - gunicorn.conf: | - secure_scheme_headers = {'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on'} - {{ if ne .Values.app.workerClass "sync" }} - worker_class = '{{ .Values.app.workerClass }}' - threads = {{ .Values.app.threads }} - {{ end }} - worker_connections = 1000 - workers = {{ .Values.app.worker }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "nomad.fullname" . }}-app - labels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-app - helm.sh/chart: {{ include "nomad.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - replicas: {{ .Values.app.replicas }} - selector: - matchLabels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-app - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-app - app.kubernetes.io/instance: {{ .Release.Name }} - {{ if .Values.roll }} - annotations: - rollme: {{ randAlphaNum 5 | quote }} - {{ end }} - spec: - containers: - - name: {{ include "nomad.name" . }}-app - image: "{{ .Values.images.nomad.name }}:{{ .Values.images.nomad.tag }}" - imagePullPolicy: {{ .Values.images.nomad.pullPolicy }} - volumeMounts: - - mountPath: /app/nomad.yaml - name: nomad-conf - subPath: nomad.yaml - - mountPath: /app/gunicorn.log.conf - name: gunicorn-log-conf - subPath: gunicorn.log.conf - - mountPath: /app/gunicorn.conf - name: gunicorn-conf - subPath: gunicorn.conf - - mountPath: /app/.volumes/fs/public - name: public-volume - - mountPath: /app/.volumes/fs/staging - name: staging-volume - - mountPath: /nomad - name: nomad-volume - env: - - name: NOMAD_SERVICE - value: "app" - - name: NOMAD_CONSOLE_LOGLEVEL - value: "{{ .Values.app.console_loglevel }}" - - name: NOMAD_LOGSTASH_LEVEL - value: "{{ .Values.app.logstash_loglevel }}" - {{ if .Values.api.apiSecret }} - - name: NOMAD_SERVICES_API_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.api.apiSecret}} - key: password - {{ end }} - {{ if .Values.keycloak.clientSecret }} - - name: NOMAD_KEYCLOAK_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.keycloak.clientSecret }} - key: password - {{ end }} - {{ if .Values.client.passwordSecret }} - - name: NOMAD_CLIENT_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.client.passwordSecret }} - key: password - {{ end }} - {{ if .Values.keycloak.passwordSecret }} - - name: NOMAD_KEYCLOAK_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.keycloak.passwordSecret }} - key: password - {{ end }} - {{ if .Values.datacite.secret }} - - name: NOMAD_DATACITE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.datacite.secret }} - key: password - - name: NOMAD_DATACITE_USER - valueFrom: - secretKeyRef: - name: {{ .Values.datacite.secret }} - key: user - {{ end }} - command: ["python", "-m", "gunicorn.app.wsgiapp", "--config", "gunicorn.conf", "--log-config", "gunicorn.log.conf", "-b 0.0.0.0:8000", "nomad.app:app"] - livenessProbe: - httpGet: - path: "{{ .Values.proxy.external.path }}/alive" - port: 8000 - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: "{{ .Values.proxy.external.path }}/alive" - port: 8000 - initialDelaySeconds: 15 - periodSeconds: 15 - timeoutSeconds: 5 - nodeSelector: - nomadtype: {{ .Values.app.nomadNodeType }} - imagePullSecrets: - - name: {{ .Values.images.secret }} - volumes: - - name: gunicorn-log-conf - configMap: - name: {{ include "nomad.fullname" . }}-app-gunicorn-log-config - - name: gunicorn-conf - configMap: - name: {{ include "nomad.fullname" . }}-app-gunicorn-config - - name: nomad-conf - configMap: - name: {{ include "nomad.fullname" . }}-configmap - - name: public-volume - hostPath: - path: {{ .Values.volumes.public }} - type: Directory - - name: staging-volume - {{ if (eq .Values.worker.storage "memory") }} - emptyDir: - medium: 'Memory' - {{ else }} - hostPath: - path: {{ .Values.volumes.staging}} - type: Directory - {{ end }} - - name: nomad-volume - hostPath: - path: {{ .Values.volumes.nomad }} - type: Directory diff --git a/ops/helm3/nomad/templates/api-service.yaml b/ops/helm3/nomad/templates/api-service.yaml deleted file mode 100644 index b2d59be505fd63a23dc53d25275d89802e466153..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/templates/api-service.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "nomad.fullname" . }}-app - labels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-app - helm.sh/chart: {{ include "nomad.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - type: ClusterIP - ports: - - port: 8000 - targetPort: 8000 - protocol: TCP - name: http - selector: - app.kubernetes.io/name: {{ include "nomad.name" . }}-app - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/ops/helm3/nomad/templates/gui-deployment.yml b/ops/helm3/nomad/templates/gui-deployment.yml deleted file mode 100644 index 4ad919a8ec009c144db4644104870ac1aa981fbd..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/templates/gui-deployment.yml +++ /dev/null @@ -1,162 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "nomad.fullname" . }}-gui-config - labels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-gui-config - helm.sh/chart: {{ include "nomad.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -data: - nginx.conf: | - server { - listen 80; - server_name www.example.com; - - location {{ .Values.proxy.external.path }} { - proxy_set_header Host $host; - proxy_pass_request_headers on; - proxy_pass http://{{ include "nomad.fullname" . }}-app:8000; - proxy_connect_timeout {{ .Values.proxy.timeout }}; - proxy_read_timeout {{ .Values.proxy.timeout }}; - } - - location {{ .Values.proxy.external.path }}/gui { - root /app/; - rewrite ^{{ .Values.proxy.external.path }}/gui/(.*)$ /nomad/$1 break; - try_files $uri {{ .Values.proxy.external.path }}/gui/index.html; - } - - location {{ .Values.proxy.external.path }}/gui/service-worker.js { - add_header Last-Modified $date_gmt; - add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0'; - if_modified_since off; - expires off; - etag off; - root /app/; - rewrite ^{{ .Values.proxy.external.path }}/gui/service-worker.js /nomad/service-worker.js break; - } - - location {{ .Values.proxy.external.path }}/gui/meta.json { - add_header Last-Modified $date_gmt; - add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0'; - if_modified_since off; - expires off; - etag off; - root /app/; - rewrite ^{{ .Values.proxy.external.path }}/gui/meta.json /nomad/meta.json break; - } - - location {{ .Values.proxy.external.path }}/api/uploads { - client_max_body_size 35g; - proxy_request_buffering off; - proxy_set_header Host $host; - proxy_pass_request_headers on; - proxy_pass http://{{ include "nomad.fullname" . }}-app:8000; - proxy_connect_timeout {{ .Values.proxy.timeout }}; - proxy_send_timeout {{ .Values.proxy.timeout }}; - } - - location {{ .Values.proxy.external.path }}/api/raw { - proxy_buffering off; - proxy_set_header Host $host; - proxy_pass_request_headers on; - proxy_pass http://{{ include "nomad.fullname" . }}-app:8000; - proxy_connect_timeout {{ .Values.proxy.timeout }}; - proxy_read_timeout {{ .Values.proxy.timeout }}; - } - - location {{ .Values.proxy.external.path }}/api/mirror { - proxy_buffering off; - proxy_set_header Host $host; - proxy_pass_request_headers on; - proxy_pass http://{{ include "nomad.fullname" . }}-app:8000; - proxy_connect_timeout {{ .Values.proxy.timeout }}; - proxy_read_timeout {{ .Values.proxy.timeout }}; - } - } - env.js: | - window.nomadEnv = { - "appBase": "{{ .Values.proxy.external.path }}", - "keycloakBase": "{{ .Values.keycloak.serverExternalUrl }}", - "keycloakRealm": "{{ .Values.keycloak.realmName }}", - "keycloakClientId": "{{ .Values.keycloak.guiClientId }}", - "matomoSiteId": {{ .Values.gui.matomoSiteId }}, - "matomoUrl": "{{ .Values.gui.matomoUrl }}", - "sendTrackingData": {{ .Values.gui.sendTrackingData }}, - "debug": {{ .Values.gui.debug }} - }; ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "nomad.fullname" . }}-gui - labels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-gui - helm.sh/chart: {{ include "nomad.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - replicas: {{ .Values.gui.replicas }} - selector: - matchLabels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-gui - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-gui - app.kubernetes.io/instance: {{ .Release.Name }} - {{ if .Values.roll }} - annotations: - rollme: {{ randAlphaNum 5 | quote }} - {{ end }} - spec: - containers: - - name: {{ include "nomad.name" . }}-gui - image: "{{ .Values.images.frontend.name }}:{{ .Values.images.frontend.tag }}" - imagePullPolicy: {{ .Values.images.frontend.pullPolicy }} - command: ["./run.sh", "{{ .Values.proxy.external.path }}"] - ports: - - containerPort: 80 - volumeMounts: - - mountPath: /etc/nginx/conf.d - readOnly: true - name: nginx-conf - - mountPath: /app/nomad/env.js - readOnly: true - subPath: env.js - name: nomad-app - - mountPath: /var/log/nginx - name: log - livenessProbe: - httpGet: - path: "{{ .Values.proxy.external.path }}/gui/index.html" - port: 80 - initialDelaySeconds: 15 - periodSeconds: 15 - readinessProbe: - httpGet: - path: "{{ .Values.proxy.external.path }}/gui/index.html" - port: 80 - initialDelaySeconds: 3 - periodSeconds: 3 - nodeSelector: - nomadtype: public - imagePullSecrets: - - name: {{ .Values.images.secret }} - volumes: - - name: nginx-conf - configMap: - name: {{ include "nomad.fullname" . }}-gui-config - items: - - key: nginx.conf - path: default.conf - - name: nomad-app - configMap: - name: {{ include "nomad.fullname" . }}-gui-config - items: - - key: env.js - path: env.js - - name: log - emptyDir: {} diff --git a/ops/helm3/nomad/templates/gui-service.yaml b/ops/helm3/nomad/templates/gui-service.yaml deleted file mode 100644 index 5d5a6e38ead7859222b9157ea283afda3561bbb8..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/templates/gui-service.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "nomad.fullname" . }}-gui - labels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-gui - helm.sh/chart: {{ include "nomad.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - {{ if .Values.nodePort }} - type: NodePort - externalIPs: - - {{ .Values.proxy.nodeIP }} - {{ end }} - ports: - - port: 80 - targetPort: 80 - protocol: TCP - name: http - {{ if .Values.nodePort }} - nodePort: {{ .Values.proxy.nodePort }} - {{ end }} - - selector: - app.kubernetes.io/name: {{ include "nomad.name" . }}-gui - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/ops/helm3/nomad/templates/nomad-configmap.yml b/ops/helm3/nomad/templates/nomad-configmap.yml deleted file mode 100644 index 55490e53eee3dfa1355d965d79f6724399a0d174..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/templates/nomad-configmap.yml +++ /dev/null @@ -1,67 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "nomad.fullname" . }}-configmap - labels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-configmap - helm.sh/chart: {{ include "nomad.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -data: - nomad.yaml: | - release: "{{ .Release.Name }}" - reprocess_unmatched: {{ .Values.reprocess_unmatched }} - fs: - tmp: "{{ .Values.volumes.tmp }}" - prefix_size: {{ .Values.volumes.prefixSize }} - working_directory: /app - logstash: - enabled: true - host: "{{ .Values.logstash.host }}" - tcp_port: {{ .Values.logstash.port }} - services: - api_host: "{{ .Values.proxy.external.host }}" - api_port: {{ .Values.proxy.external.port }} - api_base_path: "{{ .Values.proxy.external.path }}" - api_secret: "{{ .Values.api.secret }}" - https: {{ .Values.app.https }} - upload_limit: {{ .Values.api.uploadLimit }} - admin_user_id: {{ .Values.keycloak.admin_user_id }} - rabbitmq: - host: "{{ .Release.Name }}-rabbitmq" - elastic: - host: "{{ .Values.elastic.host }}" - port: {{ .Values.elastic.port }} - index_name: "{{ .Values.dbname }}" - mongo: - host: "{{ .Values.mongo.host }}" - port: {{ .Values.mongo.port }} - db_name: "{{ .Values.dbname }}" - mail: - enabled: {{ .Values.mail.enabled }} - host: "{{ .Values.mail.host }}" - {{ if .Values.mail.port }} - port: {{ .Values.mail.port }} - {{ end }} - {{ if .Values.mail.user }} - user: "{{ .Values.mail.user }}" - {{ end }} - {{ if .Values.mail.password }} - password: "{{ .Values.mail.password }}" - {{ end }} - from_address: "{{ .Values.mail.from }}" - celery: - routing: "{{ .Values.worker.routing }}" - timeout: 7200 - client: - user: "{{ .Values.client.username }}" - keycloak: - server_url: "{{ .Values.keycloak.serverUrl }}" - realm_name: "{{ .Values.keycloak.realmName }}" - username: "{{ .Values.keycloak.username }}" - client_id: "{{ .Values.keycloak.clientId }}" - normalize: - springer_db_path: "{{ .Values.springerDbPath }}" - datacite: - enabled: {{ .Values.datacite.enabled }} - prefix: "{{ .Values.datacite.prefix }}" diff --git a/ops/helm3/nomad/templates/worker-deployment.yaml b/ops/helm3/nomad/templates/worker-deployment.yaml deleted file mode 100644 index 072c8e9668fd0147be13b9f0e5266bd0f5e3475d..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/templates/worker-deployment.yaml +++ /dev/null @@ -1,126 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "nomad.fullname" . }}-worker - labels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-worker - helm.sh/chart: {{ include "nomad.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - replicas: {{ .Values.worker.replicas }} - selector: - matchLabels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-worker - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-worker - app.kubernetes.io/instance: {{ .Release.Name }} - {{ if .Values.roll }} - annotations: - rollme: {{ randAlphaNum 5 | quote }} - {{ end }} - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - topologyKey: kubernetes.io/hostname - labelSelector: - matchLabels: - app.kubernetes.io/name: {{ include "nomad.name" . }}-worker - app.kubernetes.io/instance: {{ .Release.Name }} - containers: - - name: {{ include "nomad.name" . }}-worker - image: "{{ .Values.images.nomad.name }}:{{ .Values.images.nomad.tag }}" - imagePullPolicy: {{ .Values.images.nomad.pullPolicy }} - resources: - limits: - memory: "{{ .Values.worker.memlimit }}Gi" - requests: - memory: "{{ .Values.worker.memrequest }}Gi" - volumeMounts: - - mountPath: /app/nomad.yaml - name: nomad-conf - subPath: nomad.yaml - - mountPath: /app/.volumes/fs/public - name: public-volume - - mountPath: /app/.volumes/fs/staging - name: staging-volume - - mountPath: /nomad - name: nomad-volume - env: - - name: NOMAD_SERVICE - value: "worker" - - name: NOMAD_CONSOLE_LOG_LEVEL - value: "{{ .Values.worker.console_loglevel }}" - - name: NOMAD_LOGSTASH_LEVEL - value: "{{ .Values.worker.logstash_loglevel }}" - - name: NOMAD_CELERY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{ if .Values.api.apiSecret }} - - name: NOMAD_SERVICES_API_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.api.apiSecret}} - key: password - {{ end }} - {{ if .Values.keycloak.clientSecret }} - - name: NOMAD_KEYCLOAK_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.keycloak.clientSecret }} - key: password - {{ end }} - {{ if .Values.keycloak.passwordSecret }} - - name: NOMAD_KEYCLOAK_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.keycloak.passwordSecret }} - key: password - {{ end }} - command: ["python", "-m", "celery", "worker", "-A", "nomad.processing", "-n", "$(NOMAD_CELERY_NODE_NAME)" {{ if .Values.worker.processes }}, "-c", "{{ .Values.worker.processes }}"{{ end }}] - livenessProbe: - exec: - command: - - bash - - -c - - NOMAD_LOGSTASH_LEVEL=WARNING python -m celery -A nomad.processing status | grep "$(NOMAD_CELERY_NODE_NAME):.*OK" - initialDelaySeconds: 30 - periodSeconds: 30 - readinessProbe: - exec: - command: - - bash - - -c - - NOMAD_LOGSTASH_LEVEL=WARNING python -m celery -A nomad.processing status | grep "${NOMAD_CELERY_NODE_NAME}:.*OK" - initialDelaySeconds: 5 - periodSeconds: 120 - nodeSelector: - nomadtype: {{ .Values.worker.nomadNodeType }} - imagePullSecrets: - - name: {{ .Values.images.secret }} - volumes: - - name: nomad-conf - configMap: - name: {{ include "nomad.fullname" . }}-configmap - - name: public-volume - hostPath: - path: {{ .Values.volumes.public }} - type: Directory - - name: staging-volume - {{ if (eq .Values.worker.storage "memory") }} - emptyDir: - medium: 'Memory' - {{ else }} - hostPath: - path: {{ .Values.volumes.staging}} - type: Directory - {{ end }} - - name: nomad-volume - hostPath: - path: {{ .Values.volumes.nomad }} - type: Directory diff --git a/ops/helm3/nomad/values.yaml b/ops/helm3/nomad/values.yaml deleted file mode 100644 index 287a2c80101433f34bd891c563733f6d593ce0a1..0000000000000000000000000000000000000000 --- a/ops/helm3/nomad/values.yaml +++ /dev/null @@ -1,175 +0,0 @@ -## Default values for nomad@FAIRDI - -## Everything concerning the container images to be used -images: - ## The kubernetes docker-registry secret that can be used to access the registry - # with the container image in it. - # It can be created via: - # kubectl create secret docker-registry gitlab-mpcdf --docker-server=gitlab-registry.mpcdf.mpg.de --docker-username=<your-user-name > --docker-password=<yourpass> --docker-email=<email> - secret: gitlab-mpcdf - - ## The nomad image with all nomad relavant python code. Used by api and worker service. - nomad: - ## The docker container image name without tag - name: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair - ## The docker container image tag - tag: latest - pullPolicy: IfNotPresent - - ## The nomad frontend image - frontend: - ## The docker container image name without tag - name: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend - ## The docker container image tag - tag: latest - pullPolicy: IfNotPresent - -## Ingress can be unable to provide gui and api access through kubernetes ingress -ingress: - enabled: false - annotations: - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/proxy-body-size: "32g" - nginx.ingress.kubernetes.io/proxy-request-buffering: "off" - nginx.ingress.kubernetes.io/proxy-connect-timeout: "10" - nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" - nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" - hosts: - - "" - -## Everything concerning the nomad app -app: - replicas: 1 - https: true - ## Number of gunicorn worker. - worker: 10 - ## Number of threads per gunicorn worker (for async workerClass) - threads: 4 - ## Gunircon worker class (http://docs.gunicorn.org/en/stable/settings.html#worker-class) - workerClass: 'gthread' - console_loglevel: INFO - logstash_loglevel: INFO - nomadNodeType: "public" - -## Everything concerning the nomad api -api: - ## Secret used as cryptographic seed - secret: "defaultApiSecret" - ## The adminstrator password (only way to ever set/change it) - adminPassword: "password" - ## A kubernetes secret with administrator password (supersedes adminPassword) - adminPasswordSecret: null - ## Limit of unpublished uploads per user, except admin user - uploadLimit: 10 - -## Everything concerning the nomad worker -worker: - replicas: 1 - # request and limit in GB - memrequest: 8 - memlimit: 32 - console_loglevel: ERROR - logstash_loglevel: INFO - ## There are two routing modes "queue" and "worker". The "queue" routing will use a general - # task queue and spread calc processing task over worker instances. The "worker" routing - # will send all tasks related to an upload to the same worker - routing: "queue" - storage: "disk" - nomadNodeType: "worker" - -## Everthing concerning the nomad gui -gui: - replicas: 1 - ## This variable is used in the GUI to show or hide additional information - debug: false - ## URL for matomo(piwik) user tracking - matomoUrl: 'http://nowhere.no' - ## site id for matomo(piwik) user tracking - matomoSiteId: 1 - ## send matomo(piwik) user tracking data - sendTrackingData: false - -## Everything concerning the nginx that serves the gui, proxies the api -# It is run via NodePort service -proxy: - # Set a nodePort to create a NodePort service instead of ClusterIP. Also set a nodeIP for the externalIP. - nodePort: - nodeIP: - timeout: 120 - external: - host: "labdev-nomad.esc.rzg.mpg.de" - port: 80 - path: "/fairdi/nomad/latest" - kibanaPath: "/fairdi/kibana" - -## configuration of the chart dependency for rabbitmq -rabbitmq: - persistence: - enabled: false - nodeSelector: - nomadtype: public - image.pullSecrets: nil - rabbitmq: - username: rabbitmq - password: rabbitmq - erlangCookie: SWQOKODSQALRPCLNMEQG - -## A common name/prefix for all dbs and indices. -dbname: fairdi_nomad_latest - -## The url for the upload page, used in emails to forward the user -uploadurl: 'https://labdev-nomad.esc.mpg.rzg.de/fairdi/nomad/latest/uploads' - -## Databases that are not run within the cluster. -# To run databases in the cluster, use the nomad-full helm chart. -mongo: - host: nomad-flink-01.esc - port: 27017 - -elastic: - host: nomad-flink-01.esc - port: 9200 - -logstash: - port: 5000 - host: nomad-flink-01.esc - -kibana: - port: 5601 - host: nomad-flink-01.esc - -mail: - enabled: "false" - host: 'localhost' - port: 25 - from: 'webmaster@nomad-coe.eu' - -client: - username: admin - -keycloak: - serverExternalUrl: "https://repository.nomad-coe.eu/fairdi/keycloak/auth/" - serverUrl: "https://repository.nomad-coe.eu/fairdi/keycloak/auth/" - realmName: "fairdi_nomad_test" - username: "admin" - clientId: "nomad_public" - guiClientId: "nomad_public" - admin_user_id: "00000000-0000-0000-0000-000000000000" - -## Everything concerning the data that is used by the service -volumes: - prefixSize: 2 - public: /nomad/fairdi/latest/fs/public - staging: /nomad/fairdi/latest/fs/staging - tmp: /nomad/fairdi/latest/fs/tmp - nomad: /nomad - -springerDbPath: /nomad/fairdi/db/data/springer.msg - -# Will reprocess calculations with their old matched parser, even if they do not -# match this parser anymore -reprocess_unmatched: true - -datacite: - enabled: false - prefix: "10.17172" diff --git a/run.sh b/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..d2355df51486f949aa4fdfd84e108b8095afb7c7 --- /dev/null +++ b/run.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +find gui/build -type f -exec sed -i "s_/fairdi/nomad/latest/gui_$1/gui_g" {} \; +python -m gunicorn.app.wsgiapp --config gunicorn.conf --log-config gunicorn.log.conf -b 0.0.0.0:8000 nomad.app:app \ No newline at end of file