Commit a5e94cab authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added deploy to ci. Fixed integration test, swagger after refactor. Unified helm.

parent 286b2fc0
Pipeline #72261 passed with stages
in 34 minutes and 48 seconds
**/*.git
**/.cache
**/.mypy_cache
**/__pycache__
**/*.pyc
......@@ -7,13 +8,30 @@
.pytest_cache
.vscode/
.volumes/
.git/
.coverage
data/
local/
target/
examples/
ops/
dependencies/**/test
dependencies/**/tests
data/
dependencies/**/regtests
dependencies/parsers/phonopy-library/example*
docs/.build
docs/*.graffle
.coverage
examples/
local/
target/
nomad/normalizing/data/*.db
nomad/normalizing/data/*.msg
nomad.yaml
gui/node_modules/
gui/build/
gui/public/metainfo/
gui/npm-debug.log*
gui/yarn-debug.log*
gui/yarn-error.log*
\ No newline at end of file
......@@ -22,5 +22,5 @@ target/
.vscode/
vscode/
nomad.yaml
gunicorn.log.conf
gunicorn.conf
./gunicorn.log.conf
./gunicorn.conf
# default installed image for docker executor is: python:3.6
# using an image that can do git, docker, docker-compose
image: youpy/docker-compose-git
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
# Uncomment the next lines, to run each pipline/job in its own docker environment.
# Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...).
......@@ -12,19 +12,14 @@ image: youpy/docker-compose-git
stages:
- build
- test
- integration
- release
- deploy
- release
variables:
TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:test_${CI_COMMIT_REF_NAME}
RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:test_${CI_COMMIT_REF_NAME}
FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
FRONTEND_STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:stable
FRONTEND_LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
KUBECONFIG: /etc/deploy/config
build:
......@@ -32,6 +27,7 @@ build:
before_script:
- git submodule sync
- git submodule update --init
# create the version information
- ./gitinfo.sh
script:
# ignore test directories of dependencies, there is a lot of data that we not use
......@@ -44,19 +40,6 @@ build:
- /^dev-.*$/
- tags
buildgui:
stage: build
script:
- cd gui
- ./version.sh
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $FRONTEND_TEST_IMAGE .
- docker push $FRONTEND_TEST_IMAGE
except:
- /^dev-.*$/
- tags
linting:
stage: test
image: $TEST_IMAGE
......@@ -103,6 +86,21 @@ tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
deploy:
stage: deploy
before_script:
- mkdir -p /etc/deploy
- echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- helm dependency update ops/helm/nomad
- helm upgrade --namespace nomad --install $CI_COMMIT_REF_NAME ops/helm/nomad -f ops/helm/nomad/ci-dev-values.yaml --set proxy.external.path=/dev/nomad/$CI_COMMIT_REF_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n $CI_DEV_CLUSTER_PROXY/dev/nomad/$CI_COMMIT_REF_NAME/api -u admin -w $CI_NOMAD_ADMIN_PASSWORD integrationtests --skip-doi
except:
- /^dev-.*$/
when: manual
release:
stage: release
script:
......@@ -110,9 +108,7 @@ release:
- docker pull $TEST_IMAGE
- docker tag $TEST_IMAGE $LATEST_IMAGE
- docker push $LATEST_IMAGE
- docker pull $FRONTEND_TEST_IMAGE
- docker tag $FRONTEND_TEST_IMAGE $FRONTEND_LATEST_IMAGE
- docker push $FRONTEND_LATEST_IMAGE
except:
- /^dev-.*$/
when: manual
......@@ -122,35 +118,9 @@ release_version:
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $LATEST_IMAGE
- docker tag $LATEST_IMAGE $RELEASE_IMAGE
- docker tag $LATEST_IMAGE $CI_COMMIT_TAG
- docker tag $LATEST_IMAGE $STABLE_IMAGE
- docker push $RELEASE_IMAGE
- docker push $STABLE_IMAGE
- docker pull $FRONTEND_LATEST_IMAGE
- docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_RELEASE_IMAGE
- docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_STABLE_IMAGE
- docker push $FRONTEND_RELEASE_IMAGE
- docker push $FRONTEND_STABLE_IMAGE
- docker push $CI_COMMIT_TAG
only:
- tags
## TODO This has to be fixed, we need one for testing, staging, production
# deploy_coe_staging:
# stage: deploy
# image: dtzar/helm-kubectl
# before_script:
# - mkdir -p /etc/deploy
# # kube_config is a CI/CD variable set in GitLab GUI
# - echo $CI_KUBE_CONFIG | base64 -d > /etc/deploy/config
# - helm init --upgrade
# - helm repo add stable https://kubernetes-charts.storage.googleapis.com/
# - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
# - helm repo update
# script:
# - cd ops/helm/nomad
# - export KUBECONFIG=/etc/deploy/config
# - helm dep build
# - helm upgrade nomad-coe-staging . --recreate-pods;
# except:
# - /^dev-.*$/
# when: manual
......@@ -23,7 +23,7 @@
# We use slim for the final image
FROM python:3.6-slim as final
# First, build everything in a build image
# First, build all python stuff in a python build image
FROM python:3.6-stretch as build
RUN mkdir /install
......@@ -71,10 +71,21 @@ RUN \
find /usr/local/lib/python3.6/ -name 'test' -exec rm -r '{}' + && \
find /usr/local/lib/python3.6/site-packages/ -name '*.so' -print -exec sh -c 'file "{}" | grep -q "not stripped" && strip -s "{}"' \;
# Second, create a slim final image
# Second built the GUI in a gui build image
FROM node:latest as gui_build
RUN mkdir -p /app
WORKDIR /app
ENV PATH /app/node_modules/.bin:$PATH
COPY gui/package.json /app/package.json
COPY gui/yarn.lock /app/yarn.lock
RUN yarn
COPY gui /app
RUN yarn run build
# Third, create a slim final image
FROM final
RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev
RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev curl
# copy the sources for tests, coverage, qa, etc.
COPY . /app
......@@ -89,12 +100,17 @@ RUN echo "copy 2"
COPY --from=build /install/docs/.build /app/docs/.build
RUN echo "copy 3"
# copy the nomad command
COPY --from=build /usr/local/bin/nomad /usr/local/bin/nomad
COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
RUN echo "copy 4"
# copy the gui
RUN mkdir -p /app/gui
COPY --from=gui_build /app/build /app/gui/build
RUN echo "copy 5"
RUN mkdir -p /app/.volumes/fs
RUN useradd -ms /bin/bash nomad
RUN chown -R nomad /app
RUN chmod a+rx run.sh
USER nomad
VOLUME /app/.volumes/fs
......
.. mdinclude:: ../ops/docker-compose/nomad/README.md
.. mdinclude:: ../ops/docker-compose/infrastructure/README.md
......@@ -131,39 +131,31 @@ must run to make use of nomad. We use *docker* and *docker-compose* to create a
unified environment that is easy to build and to run.
You can use *docker* to run all necessary 3rd-party components and run all nomad
services manually from your python environment. Or you can run everything within
docker containers. The former is often preferred during development, since it allows
services manually from your python environment. You can also run nomad in docker,
but using Python is often preferred during development, since it allows
you change things, debug, and re-run things quickly. The later one brings you
closer to the environment that will be used to run nomad in production.
closer to the environment that will be used to run nomad in production. For
development we recommend to skip the next step.
### Docker images for nomad
There are currently two different images and respectively two different docker files:
`Dockerfile`, and `gui/Dockerfile`.
Nomad comprises currently two services,
the *worker* (does the actual processing), and the *app*. Those services can be
run from one image that have the nomad python code and all dependencies installed. This
is covered by the `Dockerfile`.
The gui is served via containers based on the `gui/Dockerfile` which contains the
react-js frontend code. Before this image can be build, make sure to execute
is covered by the `Dockerfile` in the root directory
of the nomad sources. The gui is served also served from the *app* which entails the react-js frontend code.
Before building the image, make sure to execute
```
cd gui
./gitinfo.sh
cd ..
```
This allows to gui to present some information about the current git revision without
This allows the app to present some information about the current git revision without
having to copy the git itself to the docker build context.
The images are build via *docker-compose* and don't have to be created manually.
### Run necessary 3-rd party services with docker-compose
You can run all containers with:
```
cd ops/docker-compose/nomad
cd ops/docker-compose/infrastructure
docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d mongo elastic rabbitmq
```
......
#!/bin/sh
echo log, ref, version, commit = \"$(git log -1 --oneline)\", \"$(git describe --all)\", \"$(git describe --tags)\", \"$(git rev-parse --verify --short HEAD)\" > nomad/gitinfo.py
\ No newline at end of file
# python/backend
echo log, ref, version, commit = \"$(git log -1 --oneline)\", \"$(git describe --all)\", \"$(git describe --tags)\", \"$(git rev-parse --verify --short HEAD)\" > nomad/gitinfo.py
# gui
commit=`git rev-parse --short --verify HEAD`
sed -i -e "s/nomad-gui-commit-placeholder/$commit/g" gui/package.json
rm -f gui/package.json-e
\ No newline at end of file
.vscode/
.git/
node_modules/
build/
public/metainfo/
npm-debug.log*
yarn-debug.log*
yarn-error.log*
\ No newline at end of file
#!/bin/sh
find . -type f -exec sed -i "s_/fairdi/nomad/latest/gui_$1/gui_g" {} \;
nginx -g "daemon off;"
\ No newline at end of file
#!/bin/sh
commit=`git rev-parse --short --verify HEAD`
sed -i -e "s/nomad-gui-commit-placeholder/$commit/g" package.json
rm -f package.json-e
......@@ -34,6 +34,7 @@ from nomad import config, utils as nomad_utils
from .api import blueprint as api_blueprint, api
from .optimade import blueprint as optimade_blueprint, api as optimade
from .docs import blueprint as docs_blueprint
from .gui import blueprint as gui_blueprint
from . import common
......@@ -91,6 +92,7 @@ CORS(app)
app.register_blueprint(api_blueprint, url_prefix='/api')
app.register_blueprint(optimade_blueprint, url_prefix='/optimade')
app.register_blueprint(docs_blueprint, url_prefix='/docs')
app.register_blueprint(gui_blueprint, url_prefix='/gui')
@app.errorhandler(Exception)
......
......@@ -214,7 +214,7 @@ class ArchiveDownloadResource(Resource):
_archive_query_model = api.inherit('ArchiveSearch', search_model, {
'query': fields.Nested(query_model, description='The query used to find the requested entries.'),
'query': fields.Nested(query_model, description='The query used to find the requested entries.', skip_none=True),
'query_schema': fields.Raw(description='The query schema that defines what archive data to retrive.')
})
......@@ -225,9 +225,8 @@ class ArchiveQueryResource(Resource):
@api.response(400, 'Invalid requests, e.g. wrong owner type or bad search parameters')
@api.response(401, 'Not authorized to access the data.')
@api.response(404, 'The upload or calculation does not exist')
@api.response(200, 'Archive data send')
@api.expect(_archive_query_model)
@api.marshal_with(_archive_query_model, skip_none=True, code=200, description='Search results sent')
@api.marshal_with(_archive_query_model, skip_none=True, code=200, description='Archive search results sent')
@authenticate()
def post(self):
'''
......
......@@ -78,7 +78,7 @@ search_model_fields = {
search_model = api.model('Search', search_model_fields)
query_model_fields = {
qualified_name: fields.Raw(description=quantity.description)
qualified_name: quantity.flask_field
for qualified_name, quantity in search.search_quantities.items()}
query_model_fields.update(**{
......
......@@ -34,7 +34,7 @@ ns = api.namespace(
dataset_model = generate_flask_restplus_model(api, Dataset.m_def)
dataset_list_model = api.model('DatasetList', {
'pagination': fields.Nested(model=pagination_model),
'pagination': fields.Nested(model=pagination_model, skip_none=True),
'results': fields.List(fields.Nested(model=dataset_model, skip_none=True))
})
......
......@@ -80,7 +80,7 @@ upload_model = api.inherit('UploadProcessing', proc_model, {
})
upload_list_model = api.model('UploadList', {
'pagination': fields.Nested(model=pagination_model),
'pagination': fields.Nested(model=pagination_model, skip_none=True),
'results': fields.List(fields.Nested(model=upload_model, skip_none=True))
})
......@@ -103,7 +103,7 @@ upload_with_calcs_model = api.inherit('UploadWithPaginatedCalculations', upload_
'pagination': fields.Nested(model=api.inherit('UploadCalculationPagination', pagination_model, {
'successes': fields.Integer,
'failures': fields.Integer,
})),
}), skip_none=True),
'results': fields.List(fields.Nested(model=calc_model, skip_none=True))
}), skip_none=True)
})
......
......@@ -12,33 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# This docker image is used for the GUI container that serves the static GUI
# files.
#
# This is a multistage build, one stage for building the GUI and one final stage
# intended for the actual GUI container that serves the GUI.
# build environment
FROM node:latest as build
RUN mkdir -p /nomad/app
WORKDIR /nomad/app
ENV PATH /nomad/app/node_modules/.bin:$PATH
COPY package.json /nomad/app/package.json
COPY yarn.lock /nomad/app/yarn.lock
RUN yarn
COPY . /nomad/app
RUN yarn run build
# production environment
FROM nginx:1.13.9-alpine
COPY --from=build /nomad/app/build /app/nomad
WORKDIR /app/nomad
COPY ./run.sh run.sh
RUN chmod a+rx run.sh
CMD nginx -g "daemon off;"
VOLUME /etc/nginx/conf.d
from flask import Blueprint
import os.path
EXPOSE 8080/tcp
gui_folder = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../../gui/build'))
blueprint = Blueprint('gui', __name__, static_url_path='/', static_folder=gui_folder)
......@@ -229,59 +229,53 @@ def nginx_conf(prefix):
server {{
listen 80;
server_name www.example.com;
proxy_set_header Host $host;
location /{0} {{
return 301 /example-nomad/gui;
location / {{
proxy_pass http://app:8000;
}}
location {1}/gui {{
root /app/;
rewrite ^{1}/gui/(.*)$ /nomad/$1 break;
try_files $uri {1}/gui/index.html;
location ~ {1}\\/?(gui)?$ {{
rewrite ^ {1}/gui/ permanent;
}}
location {1}/gui/service-worker.js {{
add_header Last-Modified $date_gmt;
add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
if_modified_since off;
expires off;
etag off;
root /app/;
rewrite ^{1}/gui/service-worker.js /nomad/service-worker.js break;
location {1}/gui/ {{
proxy_intercept_errors on;
error_page 404 = @redirect_to_index;
proxy_pass http://app:8000;
}}
location @redirect_to_index {{
rewrite ^ {1}/gui/index.html break;
proxy_pass http://app:8000;
}}
location {1}/gui/meta.json {{
location ~ \\/gui\\/(service-worker\\.js|meta\\.json)$ {{
add_header Last-Modified $date_gmt;
add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
if_modified_since off;
expires off;
etag off;
root /app/;
rewrite ^{1}/gui/meta.json /nomad/meta.json break;
}}
location {1}/api {{
proxy_set_header Host $host;
proxy_pass_request_headers on;
proxy_pass http://api:8000;
proxy_pass http://app:8000;
}}
location {1}/api/uploads {{
location ~ \\/api\\/uploads\\/?$ {{
client_max_body_size 35g;
proxy_request_buffering off;
proxy_set_header Host $host;
proxy_pass_request_headers on;
proxy_pass http://api:8000;
proxy_pass http://app:8000;
}}
location ~ \\/api\\/(raw|archive) {{
proxy_buffering off;
proxy_pass http://app:8000;
}}
location {1}/api/raw {{
location ~ \\/api\\/mirror {{
proxy_buffering off;
proxy_set_header Host $host;
proxy_pass_request_headers on;
proxy_pass http://api:8000;
proxy_read_timeout 600;
proxy_pass http://app:8000;
}}
}}
'''.format(prefix.lstrip('/'), prefix))
}}'''.format(prefix))
@ops.command(help=('Generate a proxy pass config for apache2 reverse proxy servers.'))
......
......@@ -43,7 +43,7 @@ def run_app(**kwargs):
from nomad import infrastructure
from nomad.app.__main__ import run_dev_server
infrastructure.setup()
run_dev_server(port=8000, **kwargs)
run_dev_server(port=config.services.api_port, **kwargs)
def run_worker():
......
......@@ -123,11 +123,22 @@ def integrationtests(ctx, skip_parsers, skip_publish, skip_doi, skip_mirror):
assert len(search.results) <= search.pagination.total
print('performing archive paginated search')
result = client.archive.archive_query(page=1, per_page=10, **query).response().result
result = client.archive.post_archive_query(payload={
'pagination': {
'page': 1,
'per_page': 10
},
'query': query
}).response().result
assert len(result.results) > 0
print('performing archive scrolled search')
result = client.archive.archive_query(scroll=True, **query).response().result
result = client.archive.post_archive_query(payload={
'scroll': {
'scroll': True
},
'query': query
}).response().result
assert len(result.results) > 0
print('performing download')
......
......@@ -106,8 +106,6 @@ def setup_elastic():
try:
from nomad.search import entry_document
entry_document.init(index=config.elastic.index_name)
entry_document._index._name = config.elastic.index_name
logger.info('initialized elastic index', index_name=config.elastic.index_name)
except RequestError as e:
if e.status_code == 400 and 'resource_already_exists_exception' in e.error:
# happens if two services try this at the same time
......@@ -115,6 +113,9 @@ def setup_elastic():
else:
raise e
entry_document._index._name = config.elastic.index_name
logger.info('initialized elastic index', index_name=config.elastic.index_name)
return elastic_client
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment