Merge branch 'v1.0.0' into 'master'

Release v1 Closes #631, #665, #683, #670, #701, #707, #720, #722, #726, and #727 See merge request !548

Merge branch 'v1.0.0' into 'master'
Release v1 Closes #631, #665, #683, #670, #701, #707, #720, #722, #726, and #727 See merge request !548
fec41143 · Markus Scheidgen · 41a27005 · cbce03cc · fec41143 · fec41143
Commit fec41143 authored Jan 28, 2022 by Markus Scheidgen
--- a/.dockerignore
+++ b/.dockerignore
@@ -27,7 +27,7 @@ dependencies/**/tests
 dependencies/**/regtests
 dependencies/parsers/phonopy-library/example*

-docs/.build
+site
 docs/*.graffle

 nomad/normalizing/data/*.db

--- a/.gitignore
+++ b/.gitignore
@@ -21,10 +21,10 @@ test_*/
 local/
 target/
 *.swp
-*.vscode
-.vscode/
 .idea/
-vscode/
+.vscode/launch.json
+.vscode/settings.local.json
+.vscode/extensions.json
 nomad.yaml
 gunicorn.log.conf
 gunicorn.conf
@@ -34,7 +34,20 @@ setup.json
 parser.osio.log
 gui/src/metainfo.json
 gui/src/searchQuantities.json
+gui/src/toolkitMetadata.json
+gui/src/unitsData.js
+gui/src/parserMetadata.js
+gui/.editorconfig
+gui/.pnp.cjs
+gui/.yarn/
+gui/.yarnrc.yml
 examples/workdir/
 gunicorn.log.conf
 nomad/gitinfo.py
-*/node_modules/
+**/node_modules/
+jupyterhub_cookie_secret
+jupyterhub.sqlite
+jupyterhub-proxy.pid
+gui/src/parserMetadata.json
+site/
+docs/*.graffle
\ No newline at end of file
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -13,10 +13,10 @@ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner

 stages:
  - build
+  - prepare
  - test
  - release

-
 variables:
  TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
  STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
@@ -58,7 +58,14 @@ python linting:

 gui linting:
  stage: test
-  image: node
+  before_script:
+    - git submodule sync
+    - git submodule update --init -- gui/materia
+    - docker run --rm $TEST_IMAGE python -m nomad.cli dev search-quantities > gui/src/searchQuantities.json
+    - docker run --rm $TEST_IMAGE python -m nomad.cli dev units > gui/src/unitsData.json
+    - docker run --rm $TEST_IMAGE python -m nomad.cli dev toolkit-metadata > gui/src/toolkitMetadata.json
+    - docker run --rm $TEST_IMAGE python -m nomad.cli dev metainfo > gui/src/metainfo.json
+    - docker run --rm $TEST_IMAGE python -m nomad.cli dev parser-metadata > gui/src/parserMetadata.json
  script:
    - cd gui
    - yarn --network-timeout 1200000
@@ -76,11 +83,13 @@ python tests:
  services:
    - name: rabbitmq:3.7.17
      alias: rabbitmq
-    - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
+    - name: docker.elastic.co/elasticsearch/elasticsearch:6.8.15
      alias: elastic
      # fix issue with running elastic in gitlab ci runner:
      # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
      command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
+    - name: mongo:4
+      alias: mongo
  variables:
    RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
    RABBITMQ_DEFAULT_USER: rabbitmq
@@ -104,7 +113,14 @@ python tests:

 gui tests:
  stage: test
-  image: node
+  before_script:
+    - git submodule sync
+    - git submodule update --init -- gui/materia
+    - docker run --rm $TEST_IMAGE python -m nomad.cli dev search-quantities > gui/src/searchQuantities.json
+    - docker run --rm $TEST_IMAGE python -m nomad.cli dev units > gui/src/unitsData.js
+    - docker run --rm $TEST_IMAGE python -m nomad.cli dev toolkit-metadata > gui/src/toolkitMetadata.json
+    - docker run --rm $TEST_IMAGE python -m nomad.cli dev metainfo > gui/src/metainfo.json
+    - docker run --rm $TEST_IMAGE python -m nomad.cli dev parser-metadata > gui/src/parserMetadata.json
  script:
    - cd gui
    - yarn --network-timeout 1200000
@@ -134,8 +150,9 @@ install tests:
    - python -c "import nomad.datamodel, nomad.datamodel.metainfo, nomad.client"
    - pip install `echo dist/nomad-lab-*.tar.gz`[parsing]
    - python -m nomad.cli parse tests/data/parsers/vasp/vasp.xml
-    - git clone https://github.com/nomad-coe/nomad-parser-example.git example-parser
-    - python example-parser/exampleparser/__main__.py example-parser/tests/data/example.out
+    - git clone --branch v1.0.0 https://github.com/nomad-coe/nomad-parser-example.git example-parser
+    - cd example-parser
+    - python exampleparser/__main__.py tests/data/example.out
  except:
    refs:
      - tags
@@ -165,7 +182,7 @@ deploy dev:
    - mkdir -p /etc/deploy
    - echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
  script:
-    - RELEASE_NAME=`echo ${CI_COMMIT_REF_NAME} | sed -e 's/[^A-Za-z0-9\-]/-/g'`
+    - RELEASE_NAME=`echo ${CI_COMMIT_REF_NAME} | sed -e 's/[^A-Za-z0-9\-]/-/g' | sed -e 's/^[0-9-]*//g'`
    - helm dependency update ops/helm/nomad
    - helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/deployments/dev-values.yaml --set proxy.external.path=/dev/rae/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
    - docker pull $TEST_IMAGE
@@ -174,20 +191,34 @@ deploy dev:
    - /^dev-.*$/
  when: manual

-deploy prod beta:
+deploy prod:
  stage: release
  before_script:
    - mkdir -p /etc/deploy
    - echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
  script:
    - helm dependency update ops/helm/nomad
-    - helm upgrade --install nomad-beta ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
+    - helm upgrade --install nomad-prod-v1 ops/helm/nomad -f ops/helm/nomad/deployments/prod-v1-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
    - docker pull $TEST_IMAGE
-    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/beta/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests  --skip-publish --skip-doi
+    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/v1/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests  --skip-publish --skip-doi
  except:
    - /^dev-.*$/
  when: manual

+# deploy prod beta:
+#   stage: release
+#   before_script:
+#     - mkdir -p /etc/deploy
+#     - echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
+#   script:
+#     - helm dependency update ops/helm/nomad
+#     - helm upgrade --install nomad-beta ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
+#     - docker pull $TEST_IMAGE
+#     - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/beta/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests  --skip-publish --skip-doi
+#   except:
+#     - /^dev-.*$/
+#   when: manual
+
 deploy prod test:
  stage: release
  before_script:
@@ -195,9 +226,9 @@ deploy prod test:
    - echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
  script:
    - helm dependency update ops/helm/nomad
-    - helm upgrade --install nomad-test ops/helm/nomad -f ops/helm/nomad/deployments/prod-test-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
+    - helm upgrade --install nomad-test-v1 ops/helm/nomad -f ops/helm/nomad/deployments/prod-test-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
    - docker pull $TEST_IMAGE
-    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/test/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests  --skip-publish --skip-doi
+    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/v1/test/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests  --skip-publish --skip-doi
  except:
    - /^dev-.*$/
  when: manual
@@ -231,7 +262,7 @@ python package:
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - docker pull $TEST_IMAGE
-    - docker run --rm $TEST_IMAGE python -m twine upload -u gitlab-ci-token -p ${CI_JOB_TOKEN} --repository-url https://gitlab.mpcdf.mpg.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/nomad-lab.tar.gz
+    - docker run --rm $TEST_IMAGE python -m twine upload -u gitlab-ci-token -p ${CI_JOB_TOKEN} --repository-url https://gitlab.mpcdf.mpg.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/nomad-lab-*.tar.gz
  except:
    - /^dev-.*$/
  when: manual

--- a/.gitmodules
+++ b/.gitmodules
@@ -182,11 +182,8 @@
    path = dependencies/parsers/tinker
 	url = https://github.com/nomad-coe/nomad-parser-tinker.git
 	branch = master
-[submodule "dependencies/encyclopedia-gui"]
-	path = dependencies/encyclopedia-gui
-	url = https://gitlab.mpcdf.mpg.de/nomad-lab/encyclopedia-gui.git
 [submodule "dependencies/materia"]
-	path = dependencies/materia
+	path = gui/materia
 	url = https://github.com/nomad-coe/materia
 [submodule "dependencies/nomad-dos-fingerprints"]
 	path = dependencies/nomad-dos-fingerprints
@@ -204,6 +201,21 @@
 [submodule "dependencies/parsers/lobster"]
 	path = dependencies/parsers/lobster
 	url = https://github.com/ondracka/nomad_parser_lobster.git
-[submodule "dependencies/parsers/openmx"]
-	path = dependencies/parsers/openmx
-	url = https://github.com/ondracka/nomad-parser-openmx.git
+[submodule "dependencies/parsers/xps"]
+	path = dependencies/parsers/xps
+	url = https://github.com/nomad-coe/nomad-parser-xps.git
+[submodule "dependencies/parsers/aflow"]
+	path = dependencies/parsers/aflow
+	url = https://github.com/nomad-coe/nomad-parser-aflow.git
+[submodule "dependencies/parsers/mp"]
+	path = dependencies/parsers/mp
+	url = https://github.com/nomad-coe/nomad-parser-mp.git
+[submodule "dependencies/parsers/asr"]
+	path = dependencies/parsers/asr
+	url = https://github.com/nomad-coe/nomad-parser-asr.git
+[submodule "dependencies/parsers/psi4"]
+	path = dependencies/parsers/psi4
+	url = https://github.com/nomad-coe/nomad-parser-psi4.git
+[submodule "dependencies/parsers/yambo"]
+	path = dependencies/parsers/yambo
+	url = https://github.com/nomad-coe/nomad-parser-yambo.git
--- a/.vscode/javascript.json.code-snippets
+++ b/.vscode/javascript.json.code-snippets
+{
+	// Place your nomad-fairdi workspace snippets here. Each snippet is defined under a snippet name and has a scope, prefix, body and
+	// description. Add comma separated ids of the languages where the snippet is applicable in the scope field. If scope
+	// is left empty or omitted, the snippet gets applied to all languages. The prefix is what is
+	// used to trigger the snippet and the body will be expanded and inserted. Possible variables are:
+	// $1, $2 for tab stops, $0 for the final cursor position, and ${1:label}, ${2:another} for placeholders.
+	// Placeholders with the same ids are connected.
+	// Example:
+	// "Print to console": {
+	// 	"scope": "javascript,typescript",
+	// 	"prefix": "log",
+	// 	"body": [
+	// 		"console.log('$1');",
+	// 		"$2"
+	// 	],
+	// 	"description": "Log output to console"
+	// }
+	"react-component-memo-styled": {
+        "scope": "javascript",
+        "prefix": "rcms",
+        "body": [
+            "const use$1Styles = makeStyles(theme => ({",
+            "  root: {}",
+            "}))",
+            "",
+            "const $1 = React.memo(function $1({children}) {",
+            "  const classes = use$1Styles()",
+            "  return <div className={classes.root}>{children}</div>",
+            "})",
+            "$1.propTypes = {",
+            "  children: PropTypes.oneOfType([",
+            "    PropTypes.arrayOf(PropTypes.node),",
+            "    PropTypes.node",
+            "  ])",
+            "}"
+        ]
+    }
+}
\ No newline at end of file
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
+{
+    "python.pythonPath": ".pyenv/bin/python",
+    "editor.rulers": [
+        90
+    ],
+    "editor.renderWhitespace": "all",
+    "editor.tabSize": 4,
+    "[javascript]": {
+        "editor.tabSize": 2
+    },
+    "files.trimTrailingWhitespace": true,
+    "python.linting.pylintArgs": [
+        "--load-plugins=pylint_mongoengine",
+    ],
+    "python.linting.pycodestylePath": "pycodestyle",
+    "python.linting.pycodestyleEnabled": true,
+    "python.linting.pycodestyleArgs": [
+        "--ignore=E501,E701,E731"
+    ],
+    "python.linting.mypyEnabled": true,
+    "python.linting.mypyArgs": [
+        "--ignore-missing-imports",
+        "--follow-imports=silent",
+        "--no-strict-optional"
+    ],
+    "python.linting.pylintEnabled": true,
+    "files.watcherExclude": {
+      "${workspaceFolder}/.pyenv/**": true,
+    },
+    "launch": {
+        "version": "0.2.0",
+        "configurations": [
+            {
+                "name": "gui",
+                "type": "chrome",
+                "request": "launch",
+                "url": "http://localhost:3000",
+                "webRoot": "${workspaceFolder}/gui"
+            },
+            {
+                "name": "app",
+                "type": "python",
+                "request": "launch",
+                "justMyCode": false,
+                "program": "${workspaceFolder}/.pyenv/bin/nomad",
+                "args": [
+                    "admin",
+                    "run",
+                    "api"
+                ]
+            },
+            {
+                "name": "hub",
+                "type": "python",
+                "request": "launch",
+                "justMyCode": false,
+                "program": "${workspaceFolder}/.pyenv/bin/nomad",
+                "args": [
+                    "admin",
+                    "run",
+                    "hub"
+                ]
+            },
+            {
+                "name": "tests",
+                "type": "python",
+                "request": "launch",
+                "cwd": "${workspaceFolder}",
+                "program": "${workspaceFolder}/.pyenv/bin/pytest",
+                "justMyCode": false,
+                "args": [
+                    "-sv",
+                    "tests"
+                ]
+            },
+            {
+                "name": "nomad parse",
+                "type": "python",
+                "request": "launch",
+                "cwd": "${workspaceFolder}",
+                "program": "${workspaceFolder}/.env/bin/nomad",
+                "justMyCode": false,
+                "args": [
+                    "parse",
+                    "--show-archive",
+                    "tests/data/parsers/vasp/vasp.xml"
+                ]
+            },
+            {
+                "name": "Python: Current File",
+                "type": "python",
+                "request": "launch",
+                "program": "${file}"
+            },
+            {
+                "name": "Python: Attach",
+                "type": "python",
+                "request": "attach",
+                "localRoot": "${workspaceFolder}",
+                "remoteRoot": "${workspaceFolder}",
+                "port": 3000,
+                "secret": "my_secret",
+                "host": "localhost"
+            }
+        ]
+    },
+    "files.exclude": {
+        "\"**/*.pyc\": {\"when\": \"$(basename).py\"}": true,
+        "**/__pycache__": true
+    },
+    "eslint.workingDirectories": [
+        {
+            "directory": "gui"
+        }
+    ]
+}
--- a/AUTHORS
+++ b/AUTHORS
@@ -2,11 +2,5 @@ Markus Scheidgen <markus.scheidgen@gmail.com>
 Lauri Himanen <lauri.himanen@gmail.com>
 Alvin Noe Ladines <ladinesalvinnoe@gmail.com>
 Speckhard, Daniel <dts@physik.hu-berlin.de>
-Luigi Sbailo <sbailo@fhi-berlin.mpg.de>
-David Sikter <david.sikter@physik.hu-berlin.de>
-Mohammad Nakhaee <mohammad.nakhaee.1@gmail.com>
-Maja-Olivia Lenz <lenz@fhi-berlin.mpg.de>
 Martin Kuban <kuban@physik.hu-berlin.de>
-Cuauhtémoc Salazar <temok@physik.hu-berlin.de>
-Aviral Vaid <aviral@fhi-berlin.mpg.de>
-James Kermode <j.r.kermode@warwick.ac.uk>
\ No newline at end of file
+Cuauhtémoc Salazar <temok@physik.hu-berlin.de>
\ No newline at end of file
--- a/Dockerfile
+++ b/Dockerfile
@@ -69,12 +69,8 @@ WORKDIR /install
 COPY . /install
 RUN python setup.py compile
 RUN pip install .[all]
-RUN python -m nomad.cli dev metainfo > gui/src/metainfo.json
-RUN python -m nomad.cli dev search-quantities > gui/src/searchQuantities.json
-RUN python -m nomad.cli dev toolkit-metadata > gui/src/toolkitMetadata.json
-RUN python -m nomad.cli dev units > gui/src/units.js
-WORKDIR /install/docs
-RUN make html
+RUN ./generate_gui_artifacts.sh
+RUN mkdocs build && mv site docs/build
 RUN \
    find /usr/local/lib/python3.7/ -name 'tests' ! -path '*/networkx/*' -exec rm -r '{}' + && \
    find /usr/local/lib/python3.7/ -name 'test' -exec rm -r '{}' + && \
@@ -87,28 +83,16 @@ WORKDIR /app
 ENV PATH /app/node_modules/.bin:$PATH
 COPY gui/package.json /app/package.json
 COPY gui/yarn.lock /app/yarn.lock
-COPY dependencies/materia /dependencies/materia
+COPY gui/materia /app/materia
 RUN yarn
 COPY gui /app
 COPY --from=build /install/gui/src/metainfo.json /app/src/metainfo.json
 COPY --from=build /install/gui/src/searchQuantities.json /app/src/searchQuantities.json
 COPY --from=build /install/gui/src/parserMetadata.json /app/src/parserMetadata.json
 COPY --from=build /install/gui/src/toolkitMetadata.json /app/src/toolkitMetadata.json
-COPY --from=build /install/gui/src/units.js /app/src/units.js
+COPY --from=build /install/gui/src/unitsData.js /app/src/unitsData.js
 RUN yarn run build

-# Copy all sources and assets to the GUI build image, build it there, and then
-# slim down the contents before they are copied to the final image
-RUN mkdir -p /encyclopedia
-WORKDIR /encyclopedia
-COPY dependencies/encyclopedia-gui/client /encyclopedia/
-RUN npm install webpack webpack-cli
-RUN npx webpack --mode=production
-RUN rm -rf /encyclopedia/node_modules
-RUN rm -rf /encyclopedia/src
-RUN rm -f /encyclopedia/webpack.config.js
-RUN rm -f /encyclopedia/.babelrc
-
 # Third, create a slim final image
 FROM final
 RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev curl vim zip unzip
@@ -119,17 +103,14 @@ WORKDIR /app
 # transfer installed packages from dependency stage
 COPY --from=build /usr/local/lib/python3.7/site-packages /usr/local/lib/python3.7/site-packages
 # copy the documentation, its files will be served by the API
-COPY --from=build /install/docs/.build /app/docs/.build
+COPY --from=build /install/docs/build /app/docs/build
 # copy the nomad command
 COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
 # copy the gui
 RUN mkdir -p /app/gui
 COPY --from=gui_build /app/build /app/nomad/app/flask/static/gui
-# copy the encyclopedia gui production code
-COPY --from=gui_build /encyclopedia /app/nomad/app/flask/static/encyclopedia
 # remove the developer config on the gui, will be generated by run.sh from nomad.yaml
 RUN rm -f /app/nomad/app/flask/static/gui/env.js
-RUN rm -f /app/nomad/app/flask/static/encyclopedia/conf.js
 # build the python package dist
 RUN python setup.py compile
 RUN python setup.py sdist

--- a/README.md
+++ b/README.md
@@ -46,6 +46,12 @@ contributing, and API reference.

 Omitted versions are plain bugfix releases with only minor changes and fixes.

+### v1.0.0
+- new search interface
+- new v1 API (entries, materials, upload, datasets, sync)
+- refactored metainfo and parsers
+- new upload UI and incremental uploads
+
 ### v0.10.9
 - new AI Toolkit GUI page
 - many minor parser fixes and improvements

--- a/README.parsers.md
+++ b/README.parsers.md
@@ -7,27 +7,26 @@ output files and provide all information in NOMAD's unified Metainfo based Archi

 ## Preparing code input and output file for uploading to NOMAD

-NOMAD accepts `.zip` and `.tar.gz` archives as uploads. Each upload can contain arbitrary
-files and directories. NOMAD will automatically try to choose the right parser for you files.
-For each parser (i.e. for each supported code) there is one type of file that the respective
-parser can recognize. We call these files `mainfiles` as they typically are the main
-output file a code. For each `mainfile` that NOMAD discovers it will create an entry
-in the database that users can search, view, and download. NOMAD will associate all files
-in the same directory as files that also belong to that entry. Parsers
+An *upload* is basically a directory structure with files. If you have all the files locally
+you can just upload everything as a `.zip` or `.tar.gz` file in a single step. While the upload is
+in the *staging area* (i.e. before it is published) you can also easily add or remove files in the
+directory tree via the web interface. NOMAD will automatically try to choose the right parser
+for you files.
+
+For each parser there is one type of file that the respective parser can recognize. We call
+these files *mainfiles*. For each mainfile that NOMAD discovers it will create an *entry*
+in the database, which users can search, view, and download. NOMAD will consider all files
+in the same directory as *auxiliary files* that also are associated with that entry. Parsers
 might also read information from these auxillary files. This way you can add more files
-to an entry, even if the respective parser/code might not directly support it.
+to an entry, even if the respective parser/code might not use them. However, we strongly
+recommend to not have multiple mainfiles in the same directory. For CMS calculations, we
+recommend having a separate directory for each code run.

-For $codeLabel$ please provide at least the files from this table if applicable to your
-calculations (remember that you can provide more files if you want):
+For $codeLabel$ please provide at least the files from this table, if applicable
+(remember that you always can provide additional files if you want):

 $tableOfFiles$

-To create an upload with all calculations in a directory structure:
-
-```
-zip -r <upload-file>.zip <directory>/*
-```
-
 Go to the [NOMAD upload page](https://nomad-lab.eu/prod/rae/gui/uploads) to upload files
 or find instructions about how to upload files from the command line.


--- a/dependencies.sh
+++ b/dependencies.sh
@@ -5,8 +5,8 @@ set -e
 git config -f .gitmodules --get-regexp '^submodule\..*\.path$' |
    while read path_key path
    do
-        echo $path
-        (echo "$path" | grep -vEq  ^dependencies/optimade-python-tools$) \
+        (echo "$path" | grep -vEq '^dependencies/(nexus_definitions|optimade-python-tools)$') \
            && [ -f $path/requirements.txt ] && pip install -r $path/requirements.txt
        [ -f $path/setup.py ] && pip install --ignore-requires-python $1 $path
+        echo $path
    done
--- a/encyclopedia-gui @ c43f397c
+++ b/encyclopedia-gui @ c43f397c
-Subproject commit c43f397c9d65b6c3ec86c2b51b51b473576b6253
--- a/materia @ 51e2d0d8
+++ b/materia @ 51e2d0d8
-Subproject commit 51e2d0d8d30d29f0399bd669aa7cfbb9f12aa94b
--- a/nomad-dos-fingerprints @ 6e8720a1
+++ b/nomad-dos-fingerprints @ 6e8720a1
-Subproject commit f55b5ad7826793c20d93a73fab01d028fbb44b00
+Subproject commit 6e8720a1c02cb553adfaeb5b0d81b5037dbd8054
--- a/abinit @ f41d843d
+++ b/abinit @ f41d843d
-Subproject commit 226f8aa14c5ff55cd159e4386540b2a97665c3f0
+Subproject commit f41d843db1818bb06b396fffdbdf7ecd14d354ca
--- a/aflow @ e04167fc
+++ b/aflow @ e04167fc
+Subproject commit e04167fc0d1cc979a68e31ccebde3a212f52a779
--- a/amber @ ff8bb3f9
+++ b/amber @ ff8bb3f9
-Subproject commit 4bffa598eff0387e0e93d2520879426acb56763e
+Subproject commit ff8bb3f9d6c86d61bcbb817c61faa84194c06412
--- a/asap @ 6cd8ee4b
+++ b/asap @ 6cd8ee4b
-Subproject commit 9be16a520bfca38f1498bd18fc486b7981cb0e46
+Subproject commit 6cd8ee4b60893b06760d92a2b05a6745075e5612
--- a/asr @ d8afac61
+++ b/asr @ d8afac61
+Subproject commit d8afac61a197b1f662d0558676a0167570b4192b
--- a/atk @ 0bc819e9
+++ b/atk @ 0bc819e9
-Subproject commit 2e92ba01c0e62ad9e6cb8b44cefc0e2391b67a3f
+Subproject commit 0bc819e9244cc22b46c8c5370a0f1e7073499b85