Commit 86a2cd73 authored by Lauri Himanen's avatar Lauri Himanen
Browse files

First merge attempt with v0.8.0. Encyclopedia tests pass.

parents 1302ea85 47b112c3
Pipeline #72428 canceled with stages
in 3 minutes and 18 seconds
**/*.git
**/.cache
**/.mypy_cache
**/__pycache__
**/*.pyc
......@@ -7,13 +8,30 @@
.pytest_cache
.vscode/
.volumes/
.git/
.coverage
data/
local/
target/
examples/
ops/
dependencies/**/test
dependencies/**/tests
data/
dependencies/**/regtests
dependencies/parsers/phonopy-library/example*
docs/.build
docs/*.graffle
.coverage
examples/
local/
target/
nomad/normalizing/data/*.db
nomad/normalizing/data/*.msg
nomad.yaml
gui/node_modules/
gui/build/
gui/public/metainfo/
gui/npm-debug.log*
gui/yarn-debug.log*
gui/yarn-error.log*
\ No newline at end of file
.DS_Store
.*env/
.env/
.pyenv/
.pytest/
.python-version
.ipynb_checkpoints/
__pycache__
......@@ -20,6 +22,5 @@ target/
.vscode/
vscode/
nomad.yaml
gunicorn.log.conf
gunicorn.conf
nomad/normalizing/data/springer.msg
./gunicorn.log.conf
./gunicorn.conf
# default installed image for docker executor is: python:3.6
# using an image that can do git, docker, docker-compose
image: youpy/docker-compose-git
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
# Uncomment the next lines, to run each pipline/job in its own docker environment.
# Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...).
......@@ -12,19 +12,14 @@ image: youpy/docker-compose-git
stages:
- build
- test
- integration
- release
- deploy
- release
variables:
TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:test_${CI_COMMIT_REF_NAME}
RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:test_${CI_COMMIT_REF_NAME}
FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
FRONTEND_STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:stable
FRONTEND_LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
KUBECONFIG: /etc/deploy/config
build:
......@@ -32,6 +27,7 @@ build:
before_script:
- git submodule sync
- git submodule update --init --jobs=4
# create the version information
- ./gitinfo.sh
script:
# ignore test directories of dependencies, there is a lot of data that we not use
......@@ -44,26 +40,13 @@ build:
- /^dev-.*$/
- tags
buildgui:
stage: build
script:
- cd gui
- ./version.sh
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $FRONTEND_TEST_IMAGE .
- docker push $FRONTEND_TEST_IMAGE
except:
- /^dev-.*$/
- tags
linting:
stage: test
image: $TEST_IMAGE
script:
- cd /app
- python -m pycodestyle --ignore=E501,E701,E731 nomad tests
- python -m pylint --load-plugins=pylint_mongoengine nomad tests
- python -m pylint --load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin nomad tests
- python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
except:
refs:
......@@ -72,6 +55,14 @@ linting:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
gui_linting:
stage: test
image: node
script:
- cd gui
- yarn
- yarn run eslint 'src/**/*.js'
tests:
stage: test
image: $TEST_IMAGE
......@@ -91,7 +82,7 @@ tests:
NOMAD_ELASTIC_HOST: elastic
NOMAD_MONGO_HOST: mongo
NOMAD_KEYCLOAK_PASSWORD: ${CI_KEYCLOAK_ADMIN_PASSWORD}
NOMAD_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.db
NOMAD_NORMALIZE_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.msg
script:
- cd /app
- ls /builds
......@@ -103,6 +94,22 @@ tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
deploy:
stage: deploy
before_script:
- mkdir -p /etc/deploy
- echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- RELEASE_NAME=`echo ${CI_COMMIT_REF_NAME} | sed -e 's/[^A-Za-z0-9\-]/-/g'`
- helm dependency update ops/helm/nomad
- helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/ci-dev-values.yaml --set proxy.external.path=/dev/nomad/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n $CI_DEV_CLUSTER_PROXY/dev/nomad/$RELEASE_NAME/api -u admin -w $CI_NOMAD_ADMIN_PASSWORD integrationtests
except:
- /^dev-.*$/
when: manual
release:
stage: release
script:
......@@ -110,9 +117,7 @@ release:
- docker pull $TEST_IMAGE
- docker tag $TEST_IMAGE $LATEST_IMAGE
- docker push $LATEST_IMAGE
- docker pull $FRONTEND_TEST_IMAGE
- docker tag $FRONTEND_TEST_IMAGE $FRONTEND_LATEST_IMAGE
- docker push $FRONTEND_LATEST_IMAGE
except:
- /^dev-.*$/
when: manual
......@@ -122,35 +127,9 @@ release_version:
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $LATEST_IMAGE
- docker tag $LATEST_IMAGE $RELEASE_IMAGE
- docker tag $LATEST_IMAGE $CI_COMMIT_TAG
- docker tag $LATEST_IMAGE $STABLE_IMAGE
- docker push $RELEASE_IMAGE
- docker push $STABLE_IMAGE
- docker pull $FRONTEND_LATEST_IMAGE
- docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_RELEASE_IMAGE
- docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_STABLE_IMAGE
- docker push $FRONTEND_RELEASE_IMAGE
- docker push $FRONTEND_STABLE_IMAGE
- docker push $CI_COMMIT_TAG
only:
- tags
## TODO This has to be fixed, we need one for testing, staging, production
# deploy_coe_staging:
# stage: deploy
# image: dtzar/helm-kubectl
# before_script:
# - mkdir -p /etc/deploy
# # kube_config is a CI/CD variable set in GitLab GUI
# - echo $CI_KUBE_CONFIG | base64 -d > /etc/deploy/config
# - helm init --upgrade
# - helm repo add stable https://kubernetes-charts.storage.googleapis.com/
# - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
# - helm repo update
# script:
# - cd ops/helm/nomad
# - export KUBECONFIG=/etc/deploy/config
# - helm dep build
# - helm upgrade nomad-coe-staging . --recreate-pods;
# except:
# - /^dev-.*$/
# when: manual
......@@ -156,3 +156,6 @@
[submodule "dependencies/optimade-python-tools"]
path = dependencies/optimade-python-tools
url = https://github.com/markus1978/optimade-python-tools.git
[submodule "dependencies/parsers/eels"]
path = dependencies/parsers/eels
url = https://github.com/markus1978/eels.git
......@@ -23,7 +23,7 @@
# We use slim for the final image
FROM python:3.6-slim as final
# First, build everything in a build image
# First, build all python stuff in a python build image
FROM python:3.6-stretch as build
RUN mkdir /install
......@@ -37,7 +37,7 @@ RUN pip install hjson
RUN pip install scipy
RUN pip install scikit-learn==0.20.2
RUN pip install ase==3.15.0
RUN pip install Pint==0.7.2
RUN pip install Pint
RUN pip install matid
RUN pip install mdtraj==1.9.1
RUN pip install mdanalysis==0.16.2
......@@ -71,10 +71,21 @@ RUN \
find /usr/local/lib/python3.6/ -name 'test' -exec rm -r '{}' + && \
find /usr/local/lib/python3.6/site-packages/ -name '*.so' -print -exec sh -c 'file "{}" | grep -q "not stripped" && strip -s "{}"' \;
# Second, create a slim final image
# Second built the GUI in a gui build image
FROM node:latest as gui_build
RUN mkdir -p /app
WORKDIR /app
ENV PATH /app/node_modules/.bin:$PATH
COPY gui/package.json /app/package.json
COPY gui/yarn.lock /app/yarn.lock
RUN yarn
COPY gui /app
RUN yarn run build
# Third, create a slim final image
FROM final
RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev
RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev curl
# copy the sources for tests, coverage, qa, etc.
COPY . /app
......@@ -89,12 +100,17 @@ RUN echo "copy 2"
COPY --from=build /install/docs/.build /app/docs/.build
RUN echo "copy 3"
# copy the nomad command
COPY --from=build /usr/local/bin/nomad /usr/local/bin/nomad
COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
RUN echo "copy 4"
# copy the gui
RUN mkdir -p /app/gui
COPY --from=gui_build /app/build /app/gui/build
RUN echo "copy 5"
RUN mkdir -p /app/.volumes/fs
RUN useradd -ms /bin/bash nomad
RUN chown -R nomad /app
RUN chmod a+rx run.sh
USER nomad
VOLUME /app/.volumes/fs
......
......@@ -29,6 +29,15 @@ contributing, and API reference.
Omitted versions are plain bugfix releases with only minor changes and fixes.
### v0.8.0
- new underlying datamodel that allows to maintain multiple domains
- mulitple domains supported the GUI
- new metainfo implementation
- new archive based on new metainfo
- client library that serves archive data as objects (with tab completion) not dictionaries
- properties and user tab in the search GUI
- improved performance on most parsers
### v0.7.9
- Everything to run a simple NOMAD OASIS based on the central user-management
- minor bugfixes
......
#!/usr/bin/env bash
if [[ $(basename $SHELL) = 'bash' ]];
then
if [ -f ~/.bashrc ];
then
echo "Installing bash autocompletion..."
grep -q '_NOMAD_COMPLETE' ~/.bashrc
if [[ $? -ne 0 ]]; then
echo "" >> ~/.bashrc
echo 'eval "$(_NOMAD_COMPLETE=source nomad)"' >> ~/.bashrc
fi
fi
elif [[ $(basename $SHELL) = 'zsh' ]];
then
if [ -f ~/.zshrc ];
then
echo "Installing zsh autocompletion..."
grep -q 'nomad-autocompletion' ~/.zshrc
if [[ $? -ne 0 ]]; then
echo "" >> ~/.zshrc
echo "autoload bashcompinit" >> ~/.zshrc
echo "bashcompinit" >> ~/.zshrc
echo 'eval "$(_NOMAD_COMPLETE=source nomad)"' >> ~/.nomad-autocompletion.sh
echo "source ~/.nomad-autocompletion.sh" >> ~/.zshrc
fi
fi
fi
\ No newline at end of file
#!/bin/sh
set -e
working_dir=`pwd`
echo $working_dir
git config -f .gitmodules --get-regexp '^submodule\..*\.path$' |
while read path_key path
do
cd $working_dir
cd $path
if [ -z "$(git status --porcelain)" ]; then
echo "$path is clean"
else
echo "$path is not clean"
git stash
git checkout nomad-fair-metainfo
git stash apply && git add -A && git commit -a -m "$1"
git push origin nomad-fair-metainfo
fi
done
Subproject commit 0fecfb3f466690e1aa45ec6f24e9a51661e3f65c
Subproject commit 7a7c46ddeac23591b8d10aa6bbb7a7fa987df51c
Subproject commit df889f4873be90d4a0a3d0b0d3b5fca907e9561c
Subproject commit fe980c53e536f843b6e0ca85642892b8e2f75fed
Subproject commit 95b3874b72035d0529f7dd0cde419192dba68100
Subproject commit 45daba83219defb908044193214acce48baf76e9
Subproject commit 8e6def6b737a195231754f170f6df227096b00c6
Subproject commit 10f066f30370d8b7826d7cc404da360e3b3d92f2
Subproject commit d30ef0bd9275206380866c89946a0c129e7d8df9
Subproject commit f98b920976d5507c9584676e8f77071bcf041cf9
Subproject commit 42b2b0ea894aa83cda9588d3d9bf2cdee1f40a92
Subproject commit e1a03db77aa4acb3f7f55691e11f1277a2ca6726
Subproject commit c776b194dd825ad7bbe81c5c2afb1745122a1dde
Subproject commit 1cdd522775fa3ad76294bc9a58dbfcaea000f0c5
Subproject commit b87cd29be9fe1637c20a23087ce98855ff1fea44
Subproject commit 5742af2804e002b49bed42a0d5f38f8a79630697
Subproject commit a8b323f8f8b24a5cead4c7e0cca15a78193b12b5
Subproject commit e3a0ab05c403a97155ffc8bdc1a9c8321b1c4278
Subproject commit 7863212e4bde3443fae8fbb57caf3b0d95fb28f3
Subproject commit a58cfed379d240e5241bb69689441d42a5fb5e0f
Subproject commit de6d49f89748010236baa9b683464b70bc8507a6
Subproject commit 7f905180fdbcdd4e82f11a98419364f200298d51
Subproject commit 247efac74234a9c53f3a792e1027ba995b887bb6
Subproject commit 684dbf81e11c8dba24acfc5809494767059b3fcb
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment