Commit 0a6807f6 authored by speckhard's avatar speckhard
Browse files

Merged new master changes. Fixed formatting errors.

parents 66661c9a 695b74f7
Pipeline #41104 failed with stages
in 14 minutes and 14 seconds
......@@ -9,4 +9,9 @@ __pycache__
.volumes/
.pytest_cache/
.coverage
try.http
\ No newline at end of file
try.http
project/
test_*
local/
target/
*.swp
......@@ -12,6 +12,7 @@ image: youpy/docker-compose-git
stages:
- build
- test
- integration
- release
- deploy
......@@ -20,13 +21,18 @@ variables:
RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:test
FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
RAWAPI_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:test
RAWAPI_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:latest
build:
stage: build
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $TEST_IMAGE .
- docker build --no-cache -t $TEST_IMAGE .
- docker push $TEST_IMAGE
except:
- /^dev-.*$/
buildgui:
stage: build
......@@ -37,6 +43,19 @@ buildgui:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $FRONTEND_TEST_IMAGE .
- docker push $FRONTEND_TEST_IMAGE
except:
- /^dev-.*$/
buildrawapi:
stage: build
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $RAWAPI_TEST_IMAGE -f rawapi.Dockerfile .
- docker push $RAWAPI_TEST_IMAGE
except:
- /^dev-.*$/
linting:
stage: test
......@@ -46,11 +65,14 @@ linting:
- python -m pycodestyle --ignore=E501,E701 nomad tests
- python -m pylint --load-plugins=pylint_mongoengine nomad tests
- python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
except:
- /^dev-.*$/
tests:
stage: test
image: $TEST_IMAGE
services:
- postgres:latest
# this will cause a warning, as the gitlab ci runner health check will test the wrong
# port on rabbitmq container:
# https://gitlab.com/gitlab-org/gitlab-runner/issues/3163
......@@ -61,16 +83,61 @@ tests:
# https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
variables:
POSTGRES_DB: nomad
POSTGRES_USER: postgres
POSTGRES_PASSWORD: nomad
RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
RABBITMQ_DEFAULT_USER: rabbitmq
RABBITMQ_DEFAULT_PASS: rabbitmq
RABBITMQ_DEFAULT_VHOST: /
NOMAD_COE_REPO_DB_HOST: postgres
NOMAD_RABBITMQ_HOST: rabbitmq
NOMAD_ELASTIC_HOST: elastic
NOMAD_MONGO_HOST: mongo
script:
- cd /app
- python -m pytest --cov=nomad -sv tests
except:
- /^dev-.*$/
# does currently not work, current GitLab CI runner does not network services with each other
# integration-tests:
# stage: integration
# services:
# - postgres:latest
# - mongo:latest
# # this will cause a warning, as the gitlab ci runner health check will test the wrong
# # port on rabbitmq container:
# # https://gitlab.com/gitlab-org/gitlab-runner/issues/3163
# - rabbitmq
# - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
# alias: elastic
# # fix issue with running elastic in gitlab ci runner:
# # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
# command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
# - name: $TEST_IMAGE
# alias: api
# command: ["python", "-m", "nomad.client", "run", "api"]
# - name: $TEST_IMAGE
# alias: worker
# command: ["python", "-m", "nomad.client", "run", "worker"]
# variables:
# POSTGRES_DB: nomad
# POSTGRES_USER: postgres
# POSTGRES_PASSWORD: nomad
# RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
# RABBITMQ_DEFAULT_USER: rabbitmq
# RABBITMQ_DEFAULT_PASS: rabbitmq
# RABBITMQ_DEFAULT_VHOST: /
# NOMAD_COE_REPO_DB_HOST: postgres
# NOMAD_RABBITMQ_HOST: rabbitmq
# NOMAD_ELASTIC_HOST: elastic
# NOMAD_MONGO_HOST: mongo
# script:
# - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
# - integration/test_integration.sh
# except:
# - /^dev-.*$/
release:
stage: release
......@@ -82,19 +149,44 @@ release:
- docker pull $FRONTEND_TEST_IMAGE
- docker tag $FRONTEND_TEST_IMAGE $FRONTEND_RELEASE_IMAGE
- docker push $FRONTEND_RELEASE_IMAGE
only:
- master
except:
- /^dev-.*$/
when: manual
release_rawapi:
stage: release
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $RAWAPI_TEST_IMAGE
- docker tag $RAWAPI_TEST_IMAGE $RAWAPI_RELEASE_IMAGE
- docker push $RAWAPI_RELEASE_IMAGE
except:
- /^dev-.*$/
when: manual
deploy:
stage: deploy
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- cd infrastructure/nomad
- cd ops/docker-compose/nomad
# secret .env file is stored on the "production" machine and is bind mounted into the
# runner container at /nomad/config/.env
- cp /nomad/config/.env .
- docker-compose -f docker-compose.yml -f docker-compose.prod.yml build
- docker-compose -f docker-compose.yml -f docker-compose.prod.yml up --no-deps -d api worker gui
only:
- master
except:
- /^dev-.*$/
when: manual
deploy_rawapi:
stage: deploy
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- cd ops/docker-compose/rawapi
# secret .env file is stored on the "production" machine and is bind mounted into the
# runner container at /nomad/config/.env
- cp /nomad/config/.rawapi_env .env
- docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d
except:
- /^dev-.*$/
when: manual
.ropeproject/
\ No newline at end of file
.ropeproject/
*.sql
......@@ -44,7 +44,7 @@
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_utils::test_timer"
"-sv", "tests/test_api.py::test_raw_file"
]
},
{
......
......@@ -76,3 +76,5 @@ USER nomad
VOLUME /app/.volumes/fs
VOLUME /nomad
EXPOSE 8000
No preview for this file type
.build/
*.graffle/
\ No newline at end of file
This diff is collapsed.
......@@ -34,7 +34,12 @@
## nomad.user
```eval_rst
.. automodule:: nomad.user
.. automodule:: nomad.coe_repo
```
## nomad.api
```eval_rst
.. automodule:: nomad.api
```
## nomad.utils
......
......@@ -134,7 +134,7 @@ The `override` will be used automatically.
There is also an `.env` file. For development you can use `.env_development`:
```
cd ./infrastructure/nomad
cd ./ops/docker-compose/nomad
ln -s .env_development .env
```
......@@ -196,7 +196,7 @@ You can configure host ports, volume locations for host bindings, and these sort
Usually these services only used by the nomad containers, but sometimes you also
need to check something or do some manual steps.
The file `infrastructure/nomad/.env` contains variables that control the ports
The file `ops/docker-compose/nomad/.env` contains variables that control the ports
used to bind internal docker ports to your host machine. These are the ports you
have to use to connect to the respective services.
......
......@@ -2,7 +2,7 @@ import { UploadRequest } from '@navjobs/upload'
import { apiBase, appStaticBase } from './config'
const auth_headers = {
Authorization: 'Basic ' + btoa('me@gmail.com:nomad')
Authorization: 'Basic ' + btoa('sheldon.cooper@nomad-fairdi.tests.de:password')
}
const networkError = () => {
......
......@@ -48,9 +48,9 @@ class Repo extends React.Component {
})
static rowConfig = {
chemical_composition_bulk_reduced: 'Formula',
chemical_composition: 'Formula',
program_name: 'Code',
program_basis_set_type: 'Basis set',
basis_set_type: 'Basis set',
system_type: 'System',
crystal_system: 'Crystal',
space_group_number: 'Space group',
......
#!/bin/sh
set -e
echo "log into docker registry..."
docker login gitlab-registry.mpcdf.mpg.de -u $1 -p $2
echo "building images..."
cd .dependencies/nomad-lab-base
sbt "project repoTool" docker
sbt "project repoWebservice" docker
echo "pushing images..."
docker push gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/coe-repotool
docker push gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/coe-repowebservice
\ No newline at end of file
#!/bin/sh
set -e
coe_config_args="-e REPO_DB_JDBC_URL=jdbc:postgresql://postgres:5432/nomad -e REPO_ELASTIC_URL=elasticsearch://elastic:9200 --network nomad_default"
nomad="nomad -p 8000 -h localhost"
repo_tool="docker run $coe_config_args gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/coe-repotool"
echo "reset nomad"
$nomad reset
curl -XDELETE localhost:9200/repo_index
curl -XDELETE localhost:9200/repo_topics
echo "import example calculations"
$nomad upload --unstage tests/data/proc/examples_vasp.zip
echo "create a new index with coe repoTool"
$repo_tool newIndex --indexName=repo_index --indexNameTopics=repo_topics
# try to search for new calculations
curl http://localhost:8111/repo/search/calculation_groups_oldformat?query=repository_program_name%3DVASP
\ No newline at end of file
#!/bin/sh
python -m pycodestyle --ignore=E501,E701 nomad tests
python -m pylint --load-plugins=pylint_mongoengine nomad tests
python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
\ No newline at end of file
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
This module comprises the nomad@FAIRDI APIs.
The different APIs are upload, repository (raw data and search), and archive.
There is a separate documentation for the API endpoints from a client perspective.
.. autodata:: app
.. automodule:: nomad.api.app
.. automodule:: nomad.api.auth
.. automodule:: nomad.api.upload
.. automodule:: nomad.api.repository
.. automodule:: nomad.api.archive
.. automodule:: nomad.api.admin
"""
from .app import app
from . import auth, admin, upload, repository, archive, raw
@app.before_first_request
def setup():
from nomad import infrastructure
from .app import api
if not api.app.config['TESTING']:
infrastructure.setup()
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import nomad.api
if __name__ == '__main__':
nomad.api.app.run(debug=True, port=8000)
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask_restful import abort
from nomad import infrastructure
from nomad.processing import Upload
from .app import app, base_path
# TODO in production this requires authorization
@app.route('%s/admin/<string:operation>' % base_path, methods=['POST'])
def call_admin_operation(operation):
"""
Allows to perform administrative operations on the nomad services. The possible
operations are *repair_uploads*
(cleans incomplete or otherwise unexpectedly failed uploads), *reset* (clears all
databases and resets nomad).
.. :quickref: Allows to perform administrative operations on the nomad services.
:param string operation: the operation to perform
:status 400: unknown operation
:status 200: operation successfully started
:returns: an authentication token that is valid for 10 minutes.
"""
if operation == 'repair_uploads':
Upload.repair_all()
if operation == 'reset':
infrastructure.reset()
else:
abort(400, message='Unknown operation %s' % operation)
return 'done', 200
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
All APIs are served by one Flask app (:py:mod:`nomad.api.app`) under different paths.
"""
from flask import Flask, jsonify
from flask_restful import Api
from flask_cors import CORS
from werkzeug.exceptions import HTTPException
import os.path
from nomad import config
base_path = config.services.api_base_path
""" Provides the root path of the nomad APIs. """
app = Flask(
__name__,
static_url_path='%s/docs' % base_path,
static_folder=os.path.abspath(os.path.join(os.path.dirname(__file__), '../../docs/.build/html')))
""" The Flask app that serves all APIs. """
CORS(app)
api = Api(app)
""" Provides the flask restful api instance """
@app.errorhandler(HTTPException)
def handle(error):
status_code = getattr(error, 'code', 500)
name = getattr(error, 'name', 'Internal Server Error')
description = getattr(error, 'description', None)
data = dict(
code=status_code,
name=name,
description=description)
data.update(getattr(error, 'data', []))
response = jsonify(data)
response.status_code = status_code
return response
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment