Commit b38b37e9 authored by Daniel Speckhard's avatar Daniel Speckhard
Browse files

Merge + chemical labels fix.

parents d238c93f 9a313bfc
Pipeline #46505 failed with stages
in 19 minutes and 12 seconds
......@@ -12,3 +12,6 @@ __pycache__/
*.pyc
NOMAD.egg-info/
.coverage
examples/
local/
target/
......@@ -15,3 +15,5 @@ local/
target/
*.swp
*.vscode
nomad.yaml
nomad/gitinfo.py
......@@ -17,10 +17,12 @@ stages:
- deploy
variables:
TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:test_${CI_COMMIT_REF_NAME}
RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:test_${CI_COMMIT_REF_NAME}
FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
FRONTEND_LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
RAWAPI_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:${CI_COMMIT_REF_NAME}
RAWAPI_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:latest
......@@ -32,25 +34,29 @@ build:
before_script:
- git submodule sync
- git submodule update --init
- ./gitinfo.sh
script:
# ignore test directories of dependencies, there is a lot of data that we not use
# in the context of the docker image
- find dependencies -name "*test*" -type d -exec echo "{}/" \; >> .dockerignore
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build --no-cache -t $TEST_IMAGE .
- docker build -t $TEST_IMAGE .
- docker push $TEST_IMAGE
except:
- /^dev-.*$/
- tags
buildgui:
stage: build
script:
- cd gui
- ./gitinfo.sh
- ls -la src/gitinfo.json
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $FRONTEND_TEST_IMAGE .
- docker push $FRONTEND_TEST_IMAGE
except:
- /^dev-.*$/
- tags
buildrawapi:
......@@ -72,6 +78,8 @@ linting:
- python -m pylint --load-plugins=pylint_mongoengine nomad tests
- python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
except:
refs:
- tags
variables:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
......@@ -98,7 +106,7 @@ tests:
RABBITMQ_DEFAULT_USER: rabbitmq
RABBITMQ_DEFAULT_PASS: rabbitmq
RABBITMQ_DEFAULT_VHOST: /
NOMAD_COE_REPO_DB_HOST: postgres
NOMAD_REPOSITORY_DB_HOST: postgres
NOMAD_RABBITMQ_HOST: rabbitmq
NOMAD_ELASTIC_HOST: elastic
NOMAD_MONGO_HOST: mongo
......@@ -106,6 +114,8 @@ tests:
- cd /app
- python -m pytest --cov=nomad -sv tests
except:
refs:
- tags
variables:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
......@@ -139,7 +149,7 @@ tests:
# RABBITMQ_DEFAULT_USER: rabbitmq
# RABBITMQ_DEFAULT_PASS: rabbitmq
# RABBITMQ_DEFAULT_VHOST: /
# NOMAD_COE_REPO_DB_HOST: postgres
# NOMAD_REPOSITORY_DB_HOST: postgres
# NOMAD_RABBITMQ_HOST: rabbitmq
# NOMAD_ELASTIC_HOST: elastic
# NOMAD_MONGO_HOST: mongo
......@@ -154,15 +164,28 @@ release:
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $TEST_IMAGE
- docker tag $TEST_IMAGE $RELEASE_IMAGE
- docker push $RELEASE_IMAGE
- docker tag $TEST_IMAGE $LATEST_IMAGE
- docker push $LATEST_IMAGE
- docker pull $FRONTEND_TEST_IMAGE
- docker tag $FRONTEND_TEST_IMAGE $FRONTEND_RELEASE_IMAGE
- docker push $FRONTEND_RELEASE_IMAGE
- docker tag $FRONTEND_TEST_IMAGE $FRONTEND_LATEST_IMAGE
- docker push $FRONTEND_LATEST_IMAGE
except:
- /^dev-.*$/
when: manual
release_version:
stage: release
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $LATEST_IMAGE
- docker tag $LATEST_IMAGE $RELEASE_IMAGE
- docker push $RELEASE_IMAGE
- docker pull $FRONTEND_LATEST_IMAGE
- docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_RELEASE_IMAGE
- docker push $FRONTEND_RELEASE_IMAGE
only:
- tags
release_rawapi:
stage: release
script:
......@@ -194,7 +217,7 @@ deploy:
- mkdir -p /etc/deploy
# kube_config is a CI/CD variable set in GitLab GUI
- echo $CI_KUBE_CONFIG | base64 -d > /etc/deploy/config
- helm init --client-only
- helm init --upgrade
- helm repo add stable https://kubernetes-charts.storage.googleapis.com/
- helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
- helm repo update
......@@ -208,19 +231,25 @@ deploy:
- export DEPLOYS=$(helm ls | grep $RELEASE_NAME | wc -l)
- export EXTERNAL_PATH="/fairdi/nomad/v${NOMAD_VERSION}"
- export DBNAME="fairdi_nomad_v${NOMAD_VERSION//./_}"
- export FILES_PATH="/scratch/nomad-fair/fs/nomad_v${NOMAD_VERSION}"
- export FILES_PATH="/nomad/fairdi/nomad_v${NOMAD_VERSION}/fs"
- if [ ${DEPLOYS} -eq 0 ]; then
helm install --name=${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}
--set images.nomad.tag=${CI_COMMIT_REF_NAME}
--set images.frontend.tag=${CI_COMMIT_REF_NAME}
--set api.disableReset="false"
--set proxy.nodePort="300${NUMERIC_VERSION//./}"
--set proxy.external.path=${EXTERNAL_PATH}
--set dbname=${DBNAME}
--set worker.replicas=1
--set worker.memrequest=32
--set volumes.files=${FILES_PATH};
--set volumes.public=${FILES_PATH}/public
--set volumes.staging=${FILES_PATH}/staging
--set volumes.tmp=${FILES_PATH}/tmp;
else
helm upgrade ${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE} --recreate-pods;
helm upgrade ${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}
--set images.nomad.tag=${CI_COMMIT_REF_NAME}
--set images.frontend.tag=${CI_COMMIT_REF_NAME}
--recreate-pods;
fi
except:
- /^dev-.*$/
when: manual
only:
- tags
.ropeproject/
*.sql
\ No newline at end of file
*.sql
launch.json
\ No newline at end of file
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "chrome",
"request": "launch",
"name": "Launch Chrome against localhost",
"url": "http://localhost:3000",
"webRoot": "${workspaceFolder}/gui"
},
{
"name": "Python: API Flask (0.11.x or later)",
"type": "python",
"request": "launch",
"module": "flask",
"env": {
"FLASK_APP": "nomad/api.py"
},
"args": [
"run",
"--port", "8000",
"--no-debugger",
"--no-reload"
]
},
{
"name": "Python: worker nomad.processing",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/celery",
"args": [
"worker", "-l" , "debug", "-A", "nomad.processing"
]
},
{
"name": "Python: some test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_api.py::TestRepo::test_search[2-user-other_test_user]"
]
},
{
"name": "Python: crystal normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/crystal-tests/data/parsers/crystal/si.out]"
]
},
{
"name": "Python: cp2k normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/cp2k-tests/data/parsers/cp2k/si_bulk8.out]"
]
},
{
"name": "Python: cpmd normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/cpmd-tests/data/parsers/cpmd/geo_output.out]"
]
},
{
"name": "Python: wien2k normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/wien2k-tests/data/parsers/wien2k/AlN/AlN_ZB.scf]"
]
},
{
"name": "Python: gaussian normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/gaussian-tests/data/parsers/gaussian/Al.out]"
]
},
{
"name": "Python: aniline gaussian normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/gaussian-tests/data/parsers/gaussian/al-1.out]"
]
},
{
"name": "Python: test_parsing match test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_parsing.py::test_match"
]
},
{
"name": "Python: nwchem normalizer test h2o sp test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/nwchem-tests/data/parsers/nwchem/sp_output.out]"
]
},
{
"name": "Python: Vasp XML test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/vasp-tests/data/parsers/vasp/vasp.xml]"
]
},
{
"name": "Quantum Espresso Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/quantumespresso-tests/data/parsers/quantum-espresso/W.out]"
]
},
{
"name": "Abinit Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/abinit-tests/data/parsers/abinit/Fe.out]"
]
},
{
"name": "Castep Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/castep-tests/data/parsers/castep/BC2N-Pmm2-Raman.castep]"
]
},
{
"name": "DL-Poly Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/dl-poly-tests/data/parsers/dl-poly/OUTPUT]"
]
},
{
"name": "Lib Atoms Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/lib-atoms-tests/data/parsers/lib-atoms/gp.xml]"
]
},
{
"name": "Octopus Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/octopus-tests/data/parsers/octopus/stdout.txt]"
]
},
{
"name": "GULP Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/gulp-tests/data/parsers/gulp/example6.got]"
]
},
{
"name": "Turbomole Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/turbomole-tests/data/parsers/turbomole/acrolein.out]"
]
},
{
"name": "Python: Current File",
"type": "python",
"request": "launch",
"program": "${file}"
},
{
"name": "Python: Attach",
"type": "python",
"request": "attach",
"localRoot": "${workspaceFolder}",
"remoteRoot": "${workspaceFolder}",
"port": 3000,
"secret": "my_secret",
"host": "localhost"
}
]
}
\ No newline at end of file
......@@ -25,14 +25,28 @@ FROM python:3.6-slim as final
# First, build everything in a build image
FROM python:3.6-stretch as build
RUN mkdir /install
# Install some specific dependencies to make use of docker layer caching
RUN pip install --upgrade pip
RUN pip install numpy
RUN pip install cython>=0.19
RUN pip install pandas
RUN pip install h5py
RUN pip install hjson
RUN pip install scipy
RUN pip install ase==3.15.0
RUN pip install Pint==0.7.2
RUN pip install matid
RUN pip install mdtraj==1.9.1
RUN pip install mdanalysis==0.16.2
# Make will be necessary to build the docs with sphynx
RUN apt-get update && apt-get install -y make
RUN mkdir /install
WORKDIR /install
# We also install the -dev dependencies, to use this image for test and qa
RUN pip install --upgrade pip
COPY requirements.txt requirements.txt
COPY requirements.txt /install/requirements.txt
WORKDIR /install
RUN pip install -r requirements.txt
# Use docker build --build-args CACHEBUST=2 to not cache this (e.g. when you know deps have changed)
......
......@@ -52,3 +52,21 @@ python -m http.server 8888
```
Open [http://localhost:8888/html/setup.html](http://localhost:8888/html/setup.html) in
your browser.
## Change log
### v0.4.4
- support for multiple domains
- info API endpoint
### v0.4.3
- more flexible celery routing
- config via nomad.yml
- repo_db can be disabled
- publishing of calculations with failed processing
- cli for managing running processing tasks
### v0.4.2
- bugfixes regarding the migration
- better migration configurability and reproducibility
- scales to multi node kubernetes deployment
\ No newline at end of file
"""
This is a brief example demonstrating the public nomad@FAIRDI API for doing operations
that might be necessary to integrate external project data.
"""
from bravado.requests_client import RequestsClient
from bravado.client import SwaggerClient
import math
from urllib.parse import urlparse
from concurrent.futures import ThreadPoolExecutor
# nomad_url = 'http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api'
nomad_url = 'http://localhost:8000/nomad/api/'
user = 'admin'
password = 'password'
upload_file = 'external_project_example.zip'
# create the bravado client
host = urlparse(nomad_url).netloc.split(':')[0]
http_client = RequestsClient()
http_client.set_basic_auth(host, user, password)
client = SwaggerClient.from_url('%s/swagger.json' % nomad_url, http_client=http_client)
uploads = [upload.upload_id for upload in client.uploads.get_uploads().response().result]
executor = ThreadPoolExecutor(max_workers=10)
def run(upload_id):
upload = client.uploads.get_upload(upload_id=upload_id).response().result
upload_total_calcs = upload.calcs.pagination.total
per_page = 200
for page in range(1, math.ceil(upload_total_calcs / per_page) + 1):
search = client.repo.search(
page=page, per_page=per_page, order_by='mainfile',
upload_id=upload_id).response().result
print(search.pagination.page)
for upload in uploads:
executor.submit(lambda: run(upload))
#!/bin/sh
echo log, ref, version, commit = \'$(git log -1 --oneline)\', \'$(git describe --all)\', \'$(git describe)\', \'$(git rev-parse --verify HEAD)\' > nomad/gitinfo.py
\ No newline at end of file
......@@ -10,7 +10,6 @@
/build
# generated
src/gitinfo.json
public/metainfo/
# misc
......
#!/bin/sh
echo { \"log\": \"$(git log -1 --oneline)\", \"ref\": \"$(git describe --all)\", \"version\": \"$(git describe)\" } > src/gitinfo.json
\ No newline at end of file
{
"name": "nomad-fair-gui",
"version": "0.1.0",
"version": "0.4.4",
"private": true,
"dependencies": {
"@material-ui/core": "^3.9.0",
"@material-ui/icons": "^3.0.2",
"@navjobs/upload": "^3.1.3",
"autosuggest-highlight": "^3.1.1",
"base-64": "^0.1.0",
"chroma-js": "^2.0.3",
"d3": "^5.9.1",
"fetch": "^1.1.0",
"file-saver": "^2.0.0",
"html-to-react": "^1.3.3",
"marked": "^0.6.0",
"material-ui-chip-input": "^1.0.0-beta.14",
"react": "^16.4.2",
"react-autosuggest": "^9.4.3",
"react-cookie": "^3.0.8",
"react-copy-to-clipboard": "^5.0.1",
"react-dom": "^16.4.2",
......@@ -29,7 +33,6 @@
"url-parse": "^1.4.3"
},
"scripts": {
"gitinfo": "echo \"{ \\\"log\\\": \\\"$(git log -1 --oneline)\\\", \\\"ref\\\": \\\"$(git describe --all)\\\", \\\"version\\\": \\\"$(git describe)\\\" }\" > src/gitinfo.json",
"start": "react-scripts start",
"build": "react-scripts build",
"test": "react-scripts test --env=jsdom",
......
import React from 'react'
import PropTypes from 'prop-types'
import { withStyles } from '@material-ui/core/styles'
import ChipInput from 'material-ui-chip-input'
import Autosuggest from 'react-autosuggest'
import match from 'autosuggest-highlight/match'
import parse from 'autosuggest-highlight/parse'
import Paper from '@material-ui/core/Paper'
import MenuItem from '@material-ui/core/MenuItem'
function renderInput(inputProps) {
const { classes, autoFocus, value, onChange, onAdd, onDelete, chips, ref, ...other } = inputProps
return (
<ChipInput
clearInputValueOnChange
onUpdateInput={onChange}
onAdd={onAdd}
onDelete={onDelete}
value={chips}
inputRef={ref}
{...other}
/>
)
}
function renderSuggestion(suggestion, { query, isHighlighted }) {
const matches = match(getSuggestionValue(suggestion), query)
const parts = parse(getSuggestionValue(suggestion), matches)
return (
<MenuItem
selected={isHighlighted}
component='div'
onMouseDown={(e) => e.preventDefault()} // prevent the click causing the input to be blurred
>
<div>
{parts.map((part, index) => {
return part.highlight ? (
<span key={String(index)} style={{ fontWeight: 300 }}>
{part.text}
</span>
) : (
<strong key={String(index)} style={{ fontWeight: 500 }}>
{part.text}
</strong>
)
})}
</div>
</MenuItem>
)
}
function renderSuggestionsContainer(options) {
const { containerProps, children } = options
return (
<Paper {...containerProps} square>