Commit fec41143 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v1.0.0' into 'master'

Release v1

Closes #631, #665, #683, #670, #701, #707, #720, #722, #726, and #727

See merge request !548
parents 41a27005 cbce03cc
Pipeline #121030 skipped with stage
......@@ -27,7 +27,7 @@ dependencies/**/tests
dependencies/**/regtests
dependencies/parsers/phonopy-library/example*
docs/.build
site
docs/*.graffle
nomad/normalizing/data/*.db
......
......@@ -21,10 +21,10 @@ test_*/
local/
target/
*.swp
*.vscode
.vscode/
.idea/
vscode/
.vscode/launch.json
.vscode/settings.local.json
.vscode/extensions.json
nomad.yaml
gunicorn.log.conf
gunicorn.conf
......@@ -34,7 +34,20 @@ setup.json
parser.osio.log
gui/src/metainfo.json
gui/src/searchQuantities.json
gui/src/toolkitMetadata.json
gui/src/unitsData.js
gui/src/parserMetadata.js
gui/.editorconfig
gui/.pnp.cjs
gui/.yarn/
gui/.yarnrc.yml
examples/workdir/
gunicorn.log.conf
nomad/gitinfo.py
*/node_modules/
**/node_modules/
jupyterhub_cookie_secret
jupyterhub.sqlite
jupyterhub-proxy.pid
gui/src/parserMetadata.json
site/
docs/*.graffle
\ No newline at end of file
......@@ -13,10 +13,10 @@ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
stages:
- build
- prepare
- test
- release
variables:
TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
......@@ -58,7 +58,14 @@ python linting:
gui linting:
stage: test
image: node
before_script:
- git submodule sync
- git submodule update --init -- gui/materia
- docker run --rm $TEST_IMAGE python -m nomad.cli dev search-quantities > gui/src/searchQuantities.json
- docker run --rm $TEST_IMAGE python -m nomad.cli dev units > gui/src/unitsData.json
- docker run --rm $TEST_IMAGE python -m nomad.cli dev toolkit-metadata > gui/src/toolkitMetadata.json
- docker run --rm $TEST_IMAGE python -m nomad.cli dev metainfo > gui/src/metainfo.json
- docker run --rm $TEST_IMAGE python -m nomad.cli dev parser-metadata > gui/src/parserMetadata.json
script:
- cd gui
- yarn --network-timeout 1200000
......@@ -76,11 +83,13 @@ python tests:
services:
- name: rabbitmq:3.7.17
alias: rabbitmq
- name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
- name: docker.elastic.co/elasticsearch/elasticsearch:6.8.15
alias: elastic
# fix issue with running elastic in gitlab ci runner:
# https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
- name: mongo:4
alias: mongo
variables:
RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
RABBITMQ_DEFAULT_USER: rabbitmq
......@@ -104,7 +113,14 @@ python tests:
gui tests:
stage: test
image: node
before_script:
- git submodule sync
- git submodule update --init -- gui/materia
- docker run --rm $TEST_IMAGE python -m nomad.cli dev search-quantities > gui/src/searchQuantities.json
- docker run --rm $TEST_IMAGE python -m nomad.cli dev units > gui/src/unitsData.js
- docker run --rm $TEST_IMAGE python -m nomad.cli dev toolkit-metadata > gui/src/toolkitMetadata.json
- docker run --rm $TEST_IMAGE python -m nomad.cli dev metainfo > gui/src/metainfo.json
- docker run --rm $TEST_IMAGE python -m nomad.cli dev parser-metadata > gui/src/parserMetadata.json
script:
- cd gui
- yarn --network-timeout 1200000
......@@ -134,8 +150,9 @@ install tests:
- python -c "import nomad.datamodel, nomad.datamodel.metainfo, nomad.client"
- pip install `echo dist/nomad-lab-*.tar.gz`[parsing]
- python -m nomad.cli parse tests/data/parsers/vasp/vasp.xml
- git clone https://github.com/nomad-coe/nomad-parser-example.git example-parser
- python example-parser/exampleparser/__main__.py example-parser/tests/data/example.out
- git clone --branch v1.0.0 https://github.com/nomad-coe/nomad-parser-example.git example-parser
- cd example-parser
- python exampleparser/__main__.py tests/data/example.out
except:
refs:
- tags
......@@ -165,7 +182,7 @@ deploy dev:
- mkdir -p /etc/deploy
- echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- RELEASE_NAME=`echo ${CI_COMMIT_REF_NAME} | sed -e 's/[^A-Za-z0-9\-]/-/g'`
- RELEASE_NAME=`echo ${CI_COMMIT_REF_NAME} | sed -e 's/[^A-Za-z0-9\-]/-/g' | sed -e 's/^[0-9-]*//g'`
- helm dependency update ops/helm/nomad
- helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/deployments/dev-values.yaml --set proxy.external.path=/dev/rae/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
......@@ -174,20 +191,34 @@ deploy dev:
- /^dev-.*$/
when: manual
deploy prod beta:
deploy prod:
stage: release
before_script:
- mkdir -p /etc/deploy
- echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- helm dependency update ops/helm/nomad
- helm upgrade --install nomad-beta ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- helm upgrade --install nomad-prod-v1 ops/helm/nomad -f ops/helm/nomad/deployments/prod-v1-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/beta/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/v1/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
except:
- /^dev-.*$/
when: manual
# deploy prod beta:
# stage: release
# before_script:
# - mkdir -p /etc/deploy
# - echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
# script:
# - helm dependency update ops/helm/nomad
# - helm upgrade --install nomad-beta ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
# - docker pull $TEST_IMAGE
# - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/beta/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
# except:
# - /^dev-.*$/
# when: manual
deploy prod test:
stage: release
before_script:
......@@ -195,9 +226,9 @@ deploy prod test:
- echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- helm dependency update ops/helm/nomad
- helm upgrade --install nomad-test ops/helm/nomad -f ops/helm/nomad/deployments/prod-test-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- helm upgrade --install nomad-test-v1 ops/helm/nomad -f ops/helm/nomad/deployments/prod-test-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/test/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/v1/test/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
except:
- /^dev-.*$/
when: manual
......@@ -231,7 +262,7 @@ python package:
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $TEST_IMAGE
- docker run --rm $TEST_IMAGE python -m twine upload -u gitlab-ci-token -p ${CI_JOB_TOKEN} --repository-url https://gitlab.mpcdf.mpg.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/nomad-lab.tar.gz
- docker run --rm $TEST_IMAGE python -m twine upload -u gitlab-ci-token -p ${CI_JOB_TOKEN} --repository-url https://gitlab.mpcdf.mpg.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/nomad-lab-*.tar.gz
except:
- /^dev-.*$/
when: manual
......
......@@ -182,11 +182,8 @@
path = dependencies/parsers/tinker
url = https://github.com/nomad-coe/nomad-parser-tinker.git
branch = master
[submodule "dependencies/encyclopedia-gui"]
path = dependencies/encyclopedia-gui
url = https://gitlab.mpcdf.mpg.de/nomad-lab/encyclopedia-gui.git
[submodule "dependencies/materia"]
path = dependencies/materia
path = gui/materia
url = https://github.com/nomad-coe/materia
[submodule "dependencies/nomad-dos-fingerprints"]
path = dependencies/nomad-dos-fingerprints
......@@ -204,6 +201,21 @@
[submodule "dependencies/parsers/lobster"]
path = dependencies/parsers/lobster
url = https://github.com/ondracka/nomad_parser_lobster.git
[submodule "dependencies/parsers/openmx"]
path = dependencies/parsers/openmx
url = https://github.com/ondracka/nomad-parser-openmx.git
[submodule "dependencies/parsers/xps"]
path = dependencies/parsers/xps
url = https://github.com/nomad-coe/nomad-parser-xps.git
[submodule "dependencies/parsers/aflow"]
path = dependencies/parsers/aflow
url = https://github.com/nomad-coe/nomad-parser-aflow.git
[submodule "dependencies/parsers/mp"]
path = dependencies/parsers/mp
url = https://github.com/nomad-coe/nomad-parser-mp.git
[submodule "dependencies/parsers/asr"]
path = dependencies/parsers/asr
url = https://github.com/nomad-coe/nomad-parser-asr.git
[submodule "dependencies/parsers/psi4"]
path = dependencies/parsers/psi4
url = https://github.com/nomad-coe/nomad-parser-psi4.git
[submodule "dependencies/parsers/yambo"]
path = dependencies/parsers/yambo
url = https://github.com/nomad-coe/nomad-parser-yambo.git
{
// Place your nomad-fairdi workspace snippets here. Each snippet is defined under a snippet name and has a scope, prefix, body and
// description. Add comma separated ids of the languages where the snippet is applicable in the scope field. If scope
// is left empty or omitted, the snippet gets applied to all languages. The prefix is what is
// used to trigger the snippet and the body will be expanded and inserted. Possible variables are:
// $1, $2 for tab stops, $0 for the final cursor position, and ${1:label}, ${2:another} for placeholders.
// Placeholders with the same ids are connected.
// Example:
// "Print to console": {
// "scope": "javascript,typescript",
// "prefix": "log",
// "body": [
// "console.log('$1');",
// "$2"
// ],
// "description": "Log output to console"
// }
"react-component-memo-styled": {
"scope": "javascript",
"prefix": "rcms",
"body": [
"const use$1Styles = makeStyles(theme => ({",
" root: {}",
"}))",
"",
"const $1 = React.memo(function $1({children}) {",
" const classes = use$1Styles()",
" return <div className={classes.root}>{children}</div>",
"})",
"$1.propTypes = {",
" children: PropTypes.oneOfType([",
" PropTypes.arrayOf(PropTypes.node),",
" PropTypes.node",
" ])",
"}"
]
}
}
\ No newline at end of file
{
"python.pythonPath": ".pyenv/bin/python",
"editor.rulers": [
90
],
"editor.renderWhitespace": "all",
"editor.tabSize": 4,
"[javascript]": {
"editor.tabSize": 2
},
"files.trimTrailingWhitespace": true,
"python.linting.pylintArgs": [
"--load-plugins=pylint_mongoengine",
],
"python.linting.pycodestylePath": "pycodestyle",
"python.linting.pycodestyleEnabled": true,
"python.linting.pycodestyleArgs": [
"--ignore=E501,E701,E731"
],
"python.linting.mypyEnabled": true,
"python.linting.mypyArgs": [
"--ignore-missing-imports",
"--follow-imports=silent",
"--no-strict-optional"
],
"python.linting.pylintEnabled": true,
"files.watcherExclude": {
"${workspaceFolder}/.pyenv/**": true,
},
"launch": {
"version": "0.2.0",
"configurations": [
{
"name": "gui",
"type": "chrome",
"request": "launch",
"url": "http://localhost:3000",
"webRoot": "${workspaceFolder}/gui"
},
{
"name": "app",
"type": "python",
"request": "launch",
"justMyCode": false,
"program": "${workspaceFolder}/.pyenv/bin/nomad",
"args": [
"admin",
"run",
"api"
]
},
{
"name": "hub",
"type": "python",
"request": "launch",
"justMyCode": false,
"program": "${workspaceFolder}/.pyenv/bin/nomad",
"args": [
"admin",
"run",
"hub"
]
},
{
"name": "tests",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"justMyCode": false,
"args": [
"-sv",
"tests"
]
},
{
"name": "nomad parse",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.env/bin/nomad",
"justMyCode": false,
"args": [
"parse",
"--show-archive",
"tests/data/parsers/vasp/vasp.xml"
]
},
{
"name": "Python: Current File",
"type": "python",
"request": "launch",
"program": "${file}"
},
{
"name": "Python: Attach",
"type": "python",
"request": "attach",
"localRoot": "${workspaceFolder}",
"remoteRoot": "${workspaceFolder}",
"port": 3000,
"secret": "my_secret",
"host": "localhost"
}
]
},
"files.exclude": {
"\"**/*.pyc\": {\"when\": \"$(basename).py\"}": true,
"**/__pycache__": true
},
"eslint.workingDirectories": [
{
"directory": "gui"
}
]
}
......@@ -2,11 +2,5 @@ Markus Scheidgen <markus.scheidgen@gmail.com>
Lauri Himanen <lauri.himanen@gmail.com>
Alvin Noe Ladines <ladinesalvinnoe@gmail.com>
Speckhard, Daniel <dts@physik.hu-berlin.de>
Luigi Sbailo <sbailo@fhi-berlin.mpg.de>
David Sikter <david.sikter@physik.hu-berlin.de>
Mohammad Nakhaee <mohammad.nakhaee.1@gmail.com>
Maja-Olivia Lenz <lenz@fhi-berlin.mpg.de>
Martin Kuban <kuban@physik.hu-berlin.de>
Cuauhtémoc Salazar <temok@physik.hu-berlin.de>
Aviral Vaid <aviral@fhi-berlin.mpg.de>
James Kermode <j.r.kermode@warwick.ac.uk>
\ No newline at end of file
Cuauhtémoc Salazar <temok@physik.hu-berlin.de>
\ No newline at end of file
......@@ -69,12 +69,8 @@ WORKDIR /install
COPY . /install
RUN python setup.py compile
RUN pip install .[all]
RUN python -m nomad.cli dev metainfo > gui/src/metainfo.json
RUN python -m nomad.cli dev search-quantities > gui/src/searchQuantities.json
RUN python -m nomad.cli dev toolkit-metadata > gui/src/toolkitMetadata.json
RUN python -m nomad.cli dev units > gui/src/units.js
WORKDIR /install/docs
RUN make html
RUN ./generate_gui_artifacts.sh
RUN mkdocs build && mv site docs/build
RUN \
find /usr/local/lib/python3.7/ -name 'tests' ! -path '*/networkx/*' -exec rm -r '{}' + && \
find /usr/local/lib/python3.7/ -name 'test' -exec rm -r '{}' + && \
......@@ -87,28 +83,16 @@ WORKDIR /app
ENV PATH /app/node_modules/.bin:$PATH
COPY gui/package.json /app/package.json
COPY gui/yarn.lock /app/yarn.lock
COPY dependencies/materia /dependencies/materia
COPY gui/materia /app/materia
RUN yarn
COPY gui /app
COPY --from=build /install/gui/src/metainfo.json /app/src/metainfo.json
COPY --from=build /install/gui/src/searchQuantities.json /app/src/searchQuantities.json
COPY --from=build /install/gui/src/parserMetadata.json /app/src/parserMetadata.json
COPY --from=build /install/gui/src/toolkitMetadata.json /app/src/toolkitMetadata.json
COPY --from=build /install/gui/src/units.js /app/src/units.js
COPY --from=build /install/gui/src/unitsData.js /app/src/unitsData.js
RUN yarn run build
# Copy all sources and assets to the GUI build image, build it there, and then
# slim down the contents before they are copied to the final image
RUN mkdir -p /encyclopedia
WORKDIR /encyclopedia
COPY dependencies/encyclopedia-gui/client /encyclopedia/
RUN npm install webpack webpack-cli
RUN npx webpack --mode=production
RUN rm -rf /encyclopedia/node_modules
RUN rm -rf /encyclopedia/src
RUN rm -f /encyclopedia/webpack.config.js
RUN rm -f /encyclopedia/.babelrc
# Third, create a slim final image
FROM final
RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev curl vim zip unzip
......@@ -119,17 +103,14 @@ WORKDIR /app
# transfer installed packages from dependency stage
COPY --from=build /usr/local/lib/python3.7/site-packages /usr/local/lib/python3.7/site-packages
# copy the documentation, its files will be served by the API
COPY --from=build /install/docs/.build /app/docs/.build
COPY --from=build /install/docs/build /app/docs/build
# copy the nomad command
COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
# copy the gui
RUN mkdir -p /app/gui
COPY --from=gui_build /app/build /app/nomad/app/flask/static/gui
# copy the encyclopedia gui production code
COPY --from=gui_build /encyclopedia /app/nomad/app/flask/static/encyclopedia
# remove the developer config on the gui, will be generated by run.sh from nomad.yaml
RUN rm -f /app/nomad/app/flask/static/gui/env.js
RUN rm -f /app/nomad/app/flask/static/encyclopedia/conf.js
# build the python package dist
RUN python setup.py compile
RUN python setup.py sdist
......
......@@ -46,6 +46,12 @@ contributing, and API reference.
Omitted versions are plain bugfix releases with only minor changes and fixes.
### v1.0.0
- new search interface
- new v1 API (entries, materials, upload, datasets, sync)
- refactored metainfo and parsers
- new upload UI and incremental uploads
### v0.10.9
- new AI Toolkit GUI page
- many minor parser fixes and improvements
......
......@@ -7,27 +7,26 @@ output files and provide all information in NOMAD's unified Metainfo based Archi
## Preparing code input and output file for uploading to NOMAD
NOMAD accepts `.zip` and `.tar.gz` archives as uploads. Each upload can contain arbitrary
files and directories. NOMAD will automatically try to choose the right parser for you files.
For each parser (i.e. for each supported code) there is one type of file that the respective
parser can recognize. We call these files `mainfiles` as they typically are the main
output file a code. For each `mainfile` that NOMAD discovers it will create an entry
in the database that users can search, view, and download. NOMAD will associate all files
in the same directory as files that also belong to that entry. Parsers
An *upload* is basically a directory structure with files. If you have all the files locally
you can just upload everything as a `.zip` or `.tar.gz` file in a single step. While the upload is
in the *staging area* (i.e. before it is published) you can also easily add or remove files in the
directory tree via the web interface. NOMAD will automatically try to choose the right parser
for you files.
For each parser there is one type of file that the respective parser can recognize. We call
these files *mainfiles*. For each mainfile that NOMAD discovers it will create an *entry*
in the database, which users can search, view, and download. NOMAD will consider all files
in the same directory as *auxiliary files* that also are associated with that entry. Parsers
might also read information from these auxillary files. This way you can add more files
to an entry, even if the respective parser/code might not directly support it.
to an entry, even if the respective parser/code might not use them. However, we strongly
recommend to not have multiple mainfiles in the same directory. For CMS calculations, we
recommend having a separate directory for each code run.
For $codeLabel$ please provide at least the files from this table if applicable to your
calculations (remember that you can provide more files if you want):
For $codeLabel$ please provide at least the files from this table, if applicable
(remember that you always can provide additional files if you want):
$tableOfFiles$
To create an upload with all calculations in a directory structure:
```
zip -r <upload-file>.zip <directory>/*
```
Go to the [NOMAD upload page](https://nomad-lab.eu/prod/rae/gui/uploads) to upload files
or find instructions about how to upload files from the command line.
......
......@@ -5,8 +5,8 @@ set -e
git config -f .gitmodules --get-regexp '^submodule\..*\.path$' |
while read path_key path
do
echo $path
(echo "$path" | grep -vEq ^dependencies/optimade-python-tools$) \
(echo "$path" | grep -vEq '^dependencies/(nexus_definitions|optimade-python-tools)$') \
&& [ -f $path/requirements.txt ] && pip install -r $path/requirements.txt
[ -f $path/setup.py ] && pip install --ignore-requires-python $1 $path
echo $path
done
Subproject commit c43f397c9d65b6c3ec86c2b51b51b473576b6253
Subproject commit 51e2d0d8d30d29f0399bd669aa7cfbb9f12aa94b
Subproject commit f55b5ad7826793c20d93a73fab01d028fbb44b00
Subproject commit 6e8720a1c02cb553adfaeb5b0d81b5037dbd8054
Subproject commit 226f8aa14c5ff55cd159e4386540b2a97665c3f0
Subproject commit f41d843db1818bb06b396fffdbdf7ecd14d354ca
Subproject commit e04167fc0d1cc979a68e31ccebde3a212f52a779
Subproject commit 4bffa598eff0387e0e93d2520879426acb56763e
Subproject commit ff8bb3f9d6c86d61bcbb817c61faa84194c06412
Subproject commit 9be16a520bfca38f1498bd18fc486b7981cb0e46
Subproject commit 6cd8ee4b60893b06760d92a2b05a6745075e5612
Subproject commit d8afac61a197b1f662d0558676a0167570b4192b
Subproject commit 2e92ba01c0e62ad9e6cb8b44cefc0e2391b67a3f
Subproject commit 0bc819e9244cc22b46c8c5370a0f1e7073499b85
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment