Commit 86a2cd73 authored by Lauri Himanen's avatar Lauri Himanen
Browse files

First merge attempt with v0.8.0. Encyclopedia tests pass.

parents 1302ea85 47b112c3
Pipeline #72428 canceled with stages
in 3 minutes and 18 seconds
**/*.git **/*.git
**/.cache
**/.mypy_cache **/.mypy_cache
**/__pycache__ **/__pycache__
**/*.pyc **/*.pyc
...@@ -7,13 +8,30 @@ ...@@ -7,13 +8,30 @@
.pytest_cache .pytest_cache
.vscode/ .vscode/
.volumes/ .volumes/
.git/
.coverage
data/
local/
target/
examples/
ops/
dependencies/**/test dependencies/**/test
dependencies/**/tests dependencies/**/tests
data/ dependencies/**/regtests
dependencies/parsers/phonopy-library/example*
docs/.build docs/.build
docs/*.graffle docs/*.graffle
.coverage
examples/ nomad/normalizing/data/*.db
local/ nomad/normalizing/data/*.msg
target/ nomad.yaml
gui/node_modules/ gui/node_modules/
gui/build/
gui/public/metainfo/
gui/npm-debug.log*
gui/yarn-debug.log*
gui/yarn-error.log*
\ No newline at end of file
.DS_Store .DS_Store
.*env/ .env/
.pyenv/
.pytest/
.python-version .python-version
.ipynb_checkpoints/ .ipynb_checkpoints/
__pycache__ __pycache__
...@@ -20,6 +22,5 @@ target/ ...@@ -20,6 +22,5 @@ target/
.vscode/ .vscode/
vscode/ vscode/
nomad.yaml nomad.yaml
gunicorn.log.conf ./gunicorn.log.conf
gunicorn.conf ./gunicorn.conf
nomad/normalizing/data/springer.msg
# default installed image for docker executor is: python:3.6 # default installed image for docker executor is: python:3.6
# using an image that can do git, docker, docker-compose # using an image that can do git, docker, docker-compose
image: youpy/docker-compose-git image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
# Uncomment the next lines, to run each pipline/job in its own docker environment. # Uncomment the next lines, to run each pipline/job in its own docker environment.
# Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...). # Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...).
...@@ -12,19 +12,14 @@ image: youpy/docker-compose-git ...@@ -12,19 +12,14 @@ image: youpy/docker-compose-git
stages: stages:
- build - build
- test - test
- integration
- release
- deploy - deploy
- release
variables: variables:
TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:test_${CI_COMMIT_REF_NAME} TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:test_${CI_COMMIT_REF_NAME}
FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
FRONTEND_STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:stable
FRONTEND_LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
KUBECONFIG: /etc/deploy/config KUBECONFIG: /etc/deploy/config
build: build:
...@@ -32,6 +27,7 @@ build: ...@@ -32,6 +27,7 @@ build:
before_script: before_script:
- git submodule sync - git submodule sync
- git submodule update --init --jobs=4 - git submodule update --init --jobs=4
# create the version information
- ./gitinfo.sh - ./gitinfo.sh
script: script:
# ignore test directories of dependencies, there is a lot of data that we not use # ignore test directories of dependencies, there is a lot of data that we not use
...@@ -44,26 +40,13 @@ build: ...@@ -44,26 +40,13 @@ build:
- /^dev-.*$/ - /^dev-.*$/
- tags - tags
buildgui:
stage: build
script:
- cd gui
- ./version.sh
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $FRONTEND_TEST_IMAGE .
- docker push $FRONTEND_TEST_IMAGE
except:
- /^dev-.*$/
- tags
linting: linting:
stage: test stage: test
image: $TEST_IMAGE image: $TEST_IMAGE
script: script:
- cd /app - cd /app
- python -m pycodestyle --ignore=E501,E701,E731 nomad tests - python -m pycodestyle --ignore=E501,E701,E731 nomad tests
- python -m pylint --load-plugins=pylint_mongoengine nomad tests - python -m pylint --load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin nomad tests
- python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
except: except:
refs: refs:
...@@ -72,6 +55,14 @@ linting: ...@@ -72,6 +55,14 @@ linting:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/ - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
gui_linting:
stage: test
image: node
script:
- cd gui
- yarn
- yarn run eslint 'src/**/*.js'
tests: tests:
stage: test stage: test
image: $TEST_IMAGE image: $TEST_IMAGE
...@@ -91,7 +82,7 @@ tests: ...@@ -91,7 +82,7 @@ tests:
NOMAD_ELASTIC_HOST: elastic NOMAD_ELASTIC_HOST: elastic
NOMAD_MONGO_HOST: mongo NOMAD_MONGO_HOST: mongo
NOMAD_KEYCLOAK_PASSWORD: ${CI_KEYCLOAK_ADMIN_PASSWORD} NOMAD_KEYCLOAK_PASSWORD: ${CI_KEYCLOAK_ADMIN_PASSWORD}
NOMAD_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.db NOMAD_NORMALIZE_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.msg
script: script:
- cd /app - cd /app
- ls /builds - ls /builds
...@@ -103,6 +94,22 @@ tests: ...@@ -103,6 +94,22 @@ tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/ - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
deploy:
stage: deploy
before_script:
- mkdir -p /etc/deploy
- echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- RELEASE_NAME=`echo ${CI_COMMIT_REF_NAME} | sed -e 's/[^A-Za-z0-9\-]/-/g'`
- helm dependency update ops/helm/nomad
- helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/ci-dev-values.yaml --set proxy.external.path=/dev/nomad/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n $CI_DEV_CLUSTER_PROXY/dev/nomad/$RELEASE_NAME/api -u admin -w $CI_NOMAD_ADMIN_PASSWORD integrationtests
except:
- /^dev-.*$/
when: manual
release: release:
stage: release stage: release
script: script:
...@@ -110,9 +117,7 @@ release: ...@@ -110,9 +117,7 @@ release:
- docker pull $TEST_IMAGE - docker pull $TEST_IMAGE
- docker tag $TEST_IMAGE $LATEST_IMAGE - docker tag $TEST_IMAGE $LATEST_IMAGE
- docker push $LATEST_IMAGE - docker push $LATEST_IMAGE
- docker pull $FRONTEND_TEST_IMAGE
- docker tag $FRONTEND_TEST_IMAGE $FRONTEND_LATEST_IMAGE
- docker push $FRONTEND_LATEST_IMAGE
except: except:
- /^dev-.*$/ - /^dev-.*$/
when: manual when: manual
...@@ -122,35 +127,9 @@ release_version: ...@@ -122,35 +127,9 @@ release_version:
script: script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $LATEST_IMAGE - docker pull $LATEST_IMAGE
- docker tag $LATEST_IMAGE $RELEASE_IMAGE - docker tag $LATEST_IMAGE $CI_COMMIT_TAG
- docker tag $LATEST_IMAGE $STABLE_IMAGE - docker tag $LATEST_IMAGE $STABLE_IMAGE
- docker push $RELEASE_IMAGE
- docker push $STABLE_IMAGE - docker push $STABLE_IMAGE
- docker pull $FRONTEND_LATEST_IMAGE - docker push $CI_COMMIT_TAG
- docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_RELEASE_IMAGE
- docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_STABLE_IMAGE
- docker push $FRONTEND_RELEASE_IMAGE
- docker push $FRONTEND_STABLE_IMAGE
only: only:
- tags - tags
## TODO This has to be fixed, we need one for testing, staging, production
# deploy_coe_staging:
# stage: deploy
# image: dtzar/helm-kubectl
# before_script:
# - mkdir -p /etc/deploy
# # kube_config is a CI/CD variable set in GitLab GUI
# - echo $CI_KUBE_CONFIG | base64 -d > /etc/deploy/config
# - helm init --upgrade
# - helm repo add stable https://kubernetes-charts.storage.googleapis.com/
# - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
# - helm repo update
# script:
# - cd ops/helm/nomad
# - export KUBECONFIG=/etc/deploy/config
# - helm dep build
# - helm upgrade nomad-coe-staging . --recreate-pods;
# except:
# - /^dev-.*$/
# when: manual
...@@ -156,3 +156,6 @@ ...@@ -156,3 +156,6 @@
[submodule "dependencies/optimade-python-tools"] [submodule "dependencies/optimade-python-tools"]
path = dependencies/optimade-python-tools path = dependencies/optimade-python-tools
url = https://github.com/markus1978/optimade-python-tools.git url = https://github.com/markus1978/optimade-python-tools.git
[submodule "dependencies/parsers/eels"]
path = dependencies/parsers/eels
url = https://github.com/markus1978/eels.git
...@@ -23,7 +23,7 @@ ...@@ -23,7 +23,7 @@
# We use slim for the final image # We use slim for the final image
FROM python:3.6-slim as final FROM python:3.6-slim as final
# First, build everything in a build image # First, build all python stuff in a python build image
FROM python:3.6-stretch as build FROM python:3.6-stretch as build
RUN mkdir /install RUN mkdir /install
...@@ -37,7 +37,7 @@ RUN pip install hjson ...@@ -37,7 +37,7 @@ RUN pip install hjson
RUN pip install scipy RUN pip install scipy
RUN pip install scikit-learn==0.20.2 RUN pip install scikit-learn==0.20.2
RUN pip install ase==3.15.0 RUN pip install ase==3.15.0
RUN pip install Pint==0.7.2 RUN pip install Pint
RUN pip install matid RUN pip install matid
RUN pip install mdtraj==1.9.1 RUN pip install mdtraj==1.9.1
RUN pip install mdanalysis==0.16.2 RUN pip install mdanalysis==0.16.2
...@@ -71,10 +71,21 @@ RUN \ ...@@ -71,10 +71,21 @@ RUN \
find /usr/local/lib/python3.6/ -name 'test' -exec rm -r '{}' + && \ find /usr/local/lib/python3.6/ -name 'test' -exec rm -r '{}' + && \
find /usr/local/lib/python3.6/site-packages/ -name '*.so' -print -exec sh -c 'file "{}" | grep -q "not stripped" && strip -s "{}"' \; find /usr/local/lib/python3.6/site-packages/ -name '*.so' -print -exec sh -c 'file "{}" | grep -q "not stripped" && strip -s "{}"' \;
# Second, create a slim final image # Second built the GUI in a gui build image
FROM node:latest as gui_build
RUN mkdir -p /app
WORKDIR /app
ENV PATH /app/node_modules/.bin:$PATH
COPY gui/package.json /app/package.json
COPY gui/yarn.lock /app/yarn.lock
RUN yarn
COPY gui /app
RUN yarn run build
# Third, create a slim final image
FROM final FROM final
RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev curl
# copy the sources for tests, coverage, qa, etc. # copy the sources for tests, coverage, qa, etc.
COPY . /app COPY . /app
...@@ -89,12 +100,17 @@ RUN echo "copy 2" ...@@ -89,12 +100,17 @@ RUN echo "copy 2"
COPY --from=build /install/docs/.build /app/docs/.build COPY --from=build /install/docs/.build /app/docs/.build
RUN echo "copy 3" RUN echo "copy 3"
# copy the nomad command # copy the nomad command
COPY --from=build /usr/local/bin/nomad /usr/local/bin/nomad COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
RUN echo "copy 4" RUN echo "copy 4"
# copy the gui
RUN mkdir -p /app/gui
COPY --from=gui_build /app/build /app/gui/build
RUN echo "copy 5"
RUN mkdir -p /app/.volumes/fs RUN mkdir -p /app/.volumes/fs
RUN useradd -ms /bin/bash nomad RUN useradd -ms /bin/bash nomad
RUN chown -R nomad /app RUN chown -R nomad /app
RUN chmod a+rx run.sh
USER nomad USER nomad
VOLUME /app/.volumes/fs VOLUME /app/.volumes/fs
......
...@@ -29,6 +29,15 @@ contributing, and API reference. ...@@ -29,6 +29,15 @@ contributing, and API reference.
Omitted versions are plain bugfix releases with only minor changes and fixes. Omitted versions are plain bugfix releases with only minor changes and fixes.
### v0.8.0
- new underlying datamodel that allows to maintain multiple domains
- mulitple domains supported the GUI
- new metainfo implementation
- new archive based on new metainfo
- client library that serves archive data as objects (with tab completion) not dictionaries
- properties and user tab in the search GUI
- improved performance on most parsers
### v0.7.9 ### v0.7.9
- Everything to run a simple NOMAD OASIS based on the central user-management - Everything to run a simple NOMAD OASIS based on the central user-management
- minor bugfixes - minor bugfixes
......
#!/usr/bin/env bash
if [[ $(basename $SHELL) = 'bash' ]];
then
if [ -f ~/.bashrc ];
then
echo "Installing bash autocompletion..."
grep -q '_NOMAD_COMPLETE' ~/.bashrc
if [[ $? -ne 0 ]]; then
echo "" >> ~/.bashrc
echo 'eval "$(_NOMAD_COMPLETE=source nomad)"' >> ~/.bashrc
fi
fi
elif [[ $(basename $SHELL) = 'zsh' ]];
then
if [ -f ~/.zshrc ];
then
echo "Installing zsh autocompletion..."
grep -q 'nomad-autocompletion' ~/.zshrc
if [[ $? -ne 0 ]]; then
echo "" >> ~/.zshrc
echo "autoload bashcompinit" >> ~/.zshrc
echo "bashcompinit" >> ~/.zshrc
echo 'eval "$(_NOMAD_COMPLETE=source nomad)"' >> ~/.nomad-autocompletion.sh
echo "source ~/.nomad-autocompletion.sh" >> ~/.zshrc
fi
fi
fi
\ No newline at end of file
#!/bin/sh
set -e
working_dir=`pwd`
echo $working_dir
git config -f .gitmodules --get-regexp '^submodule\..*\.path$' |
while read path_key path
do
cd $working_dir
cd $path
if [ -z "$(git status --porcelain)" ]; then
echo "$path is clean"
else
echo "$path is not clean"
git stash
git checkout nomad-fair-metainfo
git stash apply && git add -A && git commit -a -m "$1"
git push origin nomad-fair-metainfo
fi
done
Subproject commit 0fecfb3f466690e1aa45ec6f24e9a51661e3f65c Subproject commit 7a7c46ddeac23591b8d10aa6bbb7a7fa987df51c
Subproject commit df889f4873be90d4a0a3d0b0d3b5fca907e9561c Subproject commit fe980c53e536f843b6e0ca85642892b8e2f75fed
Subproject commit 95b3874b72035d0529f7dd0cde419192dba68100 Subproject commit 45daba83219defb908044193214acce48baf76e9
Subproject commit 8e6def6b737a195231754f170f6df227096b00c6 Subproject commit 10f066f30370d8b7826d7cc404da360e3b3d92f2
Subproject commit d30ef0bd9275206380866c89946a0c129e7d8df9 Subproject commit f98b920976d5507c9584676e8f77071bcf041cf9
Subproject commit 42b2b0ea894aa83cda9588d3d9bf2cdee1f40a92 Subproject commit e1a03db77aa4acb3f7f55691e11f1277a2ca6726
Subproject commit c776b194dd825ad7bbe81c5c2afb1745122a1dde Subproject commit 1cdd522775fa3ad76294bc9a58dbfcaea000f0c5
Subproject commit b87cd29be9fe1637c20a23087ce98855ff1fea44 Subproject commit 5742af2804e002b49bed42a0d5f38f8a79630697
Subproject commit a8b323f8f8b24a5cead4c7e0cca15a78193b12b5 Subproject commit e3a0ab05c403a97155ffc8bdc1a9c8321b1c4278
Subproject commit 7863212e4bde3443fae8fbb57caf3b0d95fb28f3 Subproject commit a58cfed379d240e5241bb69689441d42a5fb5e0f
Subproject commit de6d49f89748010236baa9b683464b70bc8507a6 Subproject commit 7f905180fdbcdd4e82f11a98419364f200298d51
Subproject commit 247efac74234a9c53f3a792e1027ba995b887bb6 Subproject commit 684dbf81e11c8dba24acfc5809494767059b3fcb
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment