Merged new master changes. Fixed formatting errors.

.gitignore

@@ -9,4 +9,9 @@ __pycache__
 .volumes/
 .pytest_cache/
 .coverage
-try.http
\ No newline at end of file
+try.http
+project/
+test_*
+local/
+target/
+*.swp

.gitlab-ci.yml

@@ -12,6 +12,7 @@ image: youpy/docker-compose-git
 stages:
   - build
   - test
+  - integration
   - release
   - deploy
 
@@ -20,13 +21,18 @@ variables:
   RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
   FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:test
   FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
+  RAWAPI_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:test
+  RAWAPI_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:latest
 
 build:
   stage: build
   script:
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
-    - docker build -t $TEST_IMAGE .
+    - docker build --no-cache -t $TEST_IMAGE .
     - docker push $TEST_IMAGE
+  except:
+    - /^dev-.*$/
+
 
 buildgui:
   stage: build
@@ -37,6 +43,19 @@ buildgui:
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
     - docker build -t $FRONTEND_TEST_IMAGE .
     - docker push $FRONTEND_TEST_IMAGE
+  except:
+    - /^dev-.*$/
+
+
+buildrawapi:
+  stage: build
+  script:
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
+    - docker build -t $RAWAPI_TEST_IMAGE -f rawapi.Dockerfile .
+    - docker push $RAWAPI_TEST_IMAGE
+  except:
+    - /^dev-.*$/
+
 
 linting:
   stage: test
@@ -46,11 +65,14 @@ linting:
     - python -m pycodestyle --ignore=E501,E701 nomad tests
     - python -m pylint --load-plugins=pylint_mongoengine nomad tests
     - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
+  except:
+    - /^dev-.*$/
 
 tests:
   stage: test
   image: $TEST_IMAGE
   services:
+    - postgres:latest
       # this will cause a warning, as the gitlab ci runner health check will test the wrong
       # port on rabbitmq container:
       # https://gitlab.com/gitlab-org/gitlab-runner/issues/3163
@@ -61,16 +83,61 @@ tests:
       # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
       command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
   variables:
+    POSTGRES_DB: nomad
+    POSTGRES_USER: postgres
+    POSTGRES_PASSWORD: nomad
     RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
     RABBITMQ_DEFAULT_USER: rabbitmq
     RABBITMQ_DEFAULT_PASS: rabbitmq
     RABBITMQ_DEFAULT_VHOST: /
+    NOMAD_COE_REPO_DB_HOST: postgres
     NOMAD_RABBITMQ_HOST: rabbitmq
     NOMAD_ELASTIC_HOST: elastic
     NOMAD_MONGO_HOST: mongo
   script:
     - cd /app
     - python -m pytest --cov=nomad -sv tests
+  except:
+    - /^dev-.*$/
+
+# does currently not work, current GitLab CI runner does not network services with each other
+# integration-tests:
+#   stage: integration
+#   services:
+#     - postgres:latest
+#     - mongo:latest
+#       # this will cause a warning, as the gitlab ci runner health check will test the wrong
+#       # port on rabbitmq container:
+#       # https://gitlab.com/gitlab-org/gitlab-runner/issues/3163
+#     - rabbitmq
+#     - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
+#       alias: elastic
+#       # fix issue with running elastic in gitlab ci runner:
+#       # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
+#       command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
+#     - name: $TEST_IMAGE
+#       alias: api
+#       command: ["python", "-m", "nomad.client", "run", "api"]
+#     - name: $TEST_IMAGE
+#       alias: worker
+#       command: ["python", "-m", "nomad.client", "run", "worker"]
+#   variables:
+#     POSTGRES_DB: nomad
+#     POSTGRES_USER: postgres
+#     POSTGRES_PASSWORD: nomad
+#     RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
+#     RABBITMQ_DEFAULT_USER: rabbitmq
+#     RABBITMQ_DEFAULT_PASS: rabbitmq
+#     RABBITMQ_DEFAULT_VHOST: /
+#     NOMAD_COE_REPO_DB_HOST: postgres
+#     NOMAD_RABBITMQ_HOST: rabbitmq
+#     NOMAD_ELASTIC_HOST: elastic
+#     NOMAD_MONGO_HOST: mongo
+#   script:
+#     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
+#     - integration/test_integration.sh
+#   except:
+#     - /^dev-.*$/
 
 release:
   stage: release
@@ -82,19 +149,44 @@ release:
     - docker pull $FRONTEND_TEST_IMAGE
     - docker tag $FRONTEND_TEST_IMAGE $FRONTEND_RELEASE_IMAGE
     - docker push $FRONTEND_RELEASE_IMAGE
-  only:
-    - master
+  except:
+    - /^dev-.*$/
+  when: manual
+
+release_rawapi:
+  stage: release
+  script:
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
+    - docker pull $RAWAPI_TEST_IMAGE
+    - docker tag $RAWAPI_TEST_IMAGE $RAWAPI_RELEASE_IMAGE
+    - docker push $RAWAPI_RELEASE_IMAGE
+  except:
+    - /^dev-.*$/
+  when: manual
 
 deploy:
   stage: deploy
   script:
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
-    - cd infrastructure/nomad
+    - cd ops/docker-compose/nomad
     # secret .env file is stored on the "production" machine and is bind mounted into the
     # runner container at /nomad/config/.env
     - cp /nomad/config/.env .
     - docker-compose -f docker-compose.yml -f docker-compose.prod.yml build
     - docker-compose -f docker-compose.yml -f docker-compose.prod.yml up --no-deps -d api worker gui
-  only:
-    - master
+  except:
+    - /^dev-.*$/
+  when: manual
+
+deploy_rawapi:
+  stage: deploy
+  script:
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
+    - cd ops/docker-compose/rawapi
+    # secret .env file is stored on the "production" machine and is bind mounted into the
+    # runner container at /nomad/config/.env
+    - cp /nomad/config/.rawapi_env .env
+    - docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d
+  except:
+    - /^dev-.*$/
   when: manual

.vscode/.gitignore

-.ropeproject/
\ No newline at end of file
+.ropeproject/
+*.sql

.vscode/launch.json

@@ -44,7 +44,7 @@
       "cwd": "${workspaceFolder}",
       "program": "${workspaceFolder}/.pyenv/bin/pytest",
       "args": [
-        "-sv", "tests/test_utils::test_timer"
+        "-sv", "tests/test_api.py::test_raw_file"
       ]
     },
     {

Dockerfile

@@ -76,3 +76,5 @@ USER nomad
 
 VOLUME /app/.volumes/fs
 VOLUME /nomad
+
+EXPOSE 8000

docs/.gitignore

 .build/
+*.graffle/
\ No newline at end of file

docs/reference.md

@@ -34,7 +34,12 @@
 
 ## nomad.user
 ```eval_rst
-.. automodule:: nomad.user
+.. automodule:: nomad.coe_repo
+```
+
+## nomad.api
+```eval_rst
+.. automodule:: nomad.api
 ```
 
 ## nomad.utils

docs/setup.md

@@ -134,7 +134,7 @@ The `override` will be used automatically.
 
 There is also an `.env` file. For development you can use `.env_development`:
 ```
-cd ./infrastructure/nomad
+cd ./ops/docker-compose/nomad
 ln -s .env_development .env
 ```
 
@@ -196,7 +196,7 @@ You can configure host ports, volume locations for host bindings, and these sort
 Usually these services only used by the nomad containers, but sometimes you also
 need to check something or do some manual steps.
 
-The file `infrastructure/nomad/.env` contains variables that control the ports
+The file `ops/docker-compose/nomad/.env` contains variables that control the ports
 used to bind internal docker ports to your host machine. These are the ports you
 have to use to connect to the respective services.
 

gui/src/api.js

@@ -2,7 +2,7 @@ import { UploadRequest } from '@navjobs/upload'
 import { apiBase, appStaticBase } from './config'
 
 const auth_headers = {
-  Authorization: 'Basic ' + btoa('me@gmail.com:nomad')
+  Authorization: 'Basic ' + btoa('sheldon.cooper@nomad-fairdi.tests.de:password')
 }
 
 const networkError = () => {

gui/src/components/Repo.js

@@ -48,9 +48,9 @@ class Repo extends React.Component {
   })
 
   static rowConfig = {
-    chemical_composition_bulk_reduced: 'Formula',
+    chemical_composition: 'Formula',
     program_name: 'Code',
-    program_basis_set_type: 'Basis set',
+    basis_set_type: 'Basis set',
     system_type: 'System',
     crystal_system: 'Crystal',
     space_group_number: 'Space group',

integration/build_coe_container.sh

+#!/bin/sh
+
+set -e
+
+echo "log into docker registry..."
+docker login gitlab-registry.mpcdf.mpg.de -u $1 -p $2
+
+echo "building images..."
+cd .dependencies/nomad-lab-base
+sbt "project repoTool" docker
+sbt "project repoWebservice" docker
+
+echo "pushing images..."
+docker push gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/coe-repotool
+docker push gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/coe-repowebservice
\ No newline at end of file

integration/test_integration.sh

+#!/bin/sh
+set -e
+
+coe_config_args="-e REPO_DB_JDBC_URL=jdbc:postgresql://postgres:5432/nomad -e REPO_ELASTIC_URL=elasticsearch://elastic:9200 --network nomad_default"
+
+nomad="nomad -p 8000 -h localhost"
+repo_tool="docker run $coe_config_args gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/coe-repotool"
+
+echo "reset nomad"
+$nomad reset
+curl -XDELETE localhost:9200/repo_index
+curl -XDELETE localhost:9200/repo_topics
+
+echo "import example calculations"
+$nomad upload --unstage tests/data/proc/examples_vasp.zip
+
+echo "create a new index with coe repoTool"
+$repo_tool newIndex --indexName=repo_index --indexNameTopics=repo_topics
+
+# try to search for new calculations
+curl http://localhost:8111/repo/search/calculation_groups_oldformat?query=repository_program_name%3DVASP
\ No newline at end of file

linting.sh

-#!/bin/sh
-
-python -m pycodestyle --ignore=E501,E701 nomad tests
-python -m pylint --load-plugins=pylint_mongoengine nomad tests
-python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
\ No newline at end of file

nomad/api/__init__.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module comprises the nomad@FAIRDI APIs.
+
+The different APIs are upload, repository (raw data and search), and archive.
+
+There is a separate documentation for the API endpoints from a client perspective.
+
+.. autodata:: app
+
+.. automodule:: nomad.api.app
+.. automodule:: nomad.api.auth
+.. automodule:: nomad.api.upload
+.. automodule:: nomad.api.repository
+.. automodule:: nomad.api.archive
+.. automodule:: nomad.api.admin
+"""
+from .app import app
+from . import auth, admin, upload, repository, archive, raw
+
+
+@app.before_first_request
+def setup():
+    from nomad import infrastructure
+    from .app import api
+
+    if not api.app.config['TESTING']:
+        infrastructure.setup()

nomad/api/__main__.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import nomad.api
+
+
+if __name__ == '__main__':
+    nomad.api.app.run(debug=True, port=8000)

nomad/api/admin.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask_restful import abort
+
+from nomad import infrastructure
+from nomad.processing import Upload
+
+from .app import app, base_path
+
+
+# TODO in production this requires authorization
+@app.route('%s/admin/<string:operation>' % base_path, methods=['POST'])
+def call_admin_operation(operation):
+    """
+    Allows to perform administrative operations on the nomad services. The possible
+    operations are *repair_uploads*
+    (cleans incomplete or otherwise unexpectedly failed uploads), *reset* (clears all
+    databases and resets nomad).
+
+    .. :quickref: Allows to perform administrative operations on the nomad services.
+
+    :param string operation: the operation to perform
+    :status 400: unknown operation
+    :status 200: operation successfully started
+    :returns: an authentication token that is valid for 10 minutes.
+    """
+    if operation == 'repair_uploads':
+        Upload.repair_all()
+    if operation == 'reset':
+        infrastructure.reset()
+    else:
+        abort(400, message='Unknown operation %s' % operation)
+
+    return 'done', 200

nomad/api/app.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+All APIs are served by one Flask app (:py:mod:`nomad.api.app`) under different paths.
+"""
+
+from flask import Flask, jsonify
+from flask_restful import Api
+from flask_cors import CORS
+from werkzeug.exceptions import HTTPException
+import os.path
+
+from nomad import config
+
+base_path = config.services.api_base_path
+""" Provides the root path of the nomad APIs. """
+
+app = Flask(
+    __name__,
+    static_url_path='%s/docs' % base_path,
+    static_folder=os.path.abspath(os.path.join(os.path.dirname(__file__), '../../docs/.build/html')))
+""" The Flask app that serves all APIs. """
+
+CORS(app)
+
+api = Api(app)
+""" Provides the flask restful api instance """
+
+
+@app.errorhandler(HTTPException)
+def handle(error):
+    status_code = getattr(error, 'code', 500)
+    name = getattr(error, 'name', 'Internal Server Error')
+    description = getattr(error, 'description', None)
+    data = dict(
+        code=status_code,
+        name=name,
+        description=description)
+    data.update(getattr(error, 'data', []))
+    response = jsonify(data)
+    response.status_code = status_code
+    return response