diff --git a/.vscode/.gitignore b/.vscode/.gitignore
index 08cf7faf447e96e9a9e8ff243ee4e84ddb596209..3f92a7033a197c6e6a1b11278b1a8cd1aa18e6ca 100644
--- a/.vscode/.gitignore
+++ b/.vscode/.gitignore
@@ -1,2 +1,3 @@
.ropeproject/
-*.sql
\ No newline at end of file
+*.sql
+launch.json
\ No newline at end of file
diff --git a/.vscode/launch.json b/.vscode/launch.json
deleted file mode 100644
index a8858ee1adebe5cbbb00e99e68f7c2ba0fdddfc3..0000000000000000000000000000000000000000
--- a/.vscode/launch.json
+++ /dev/null
@@ -1,217 +0,0 @@
-{
- // Use IntelliSense to learn about possible attributes.
- // Hover to view descriptions of existing attributes.
- // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
- "version": "0.2.0",
- "configurations": [
-
- {
- "type": "chrome",
- "request": "launch",
- "name": "Launch Chrome against localhost",
- "url": "http://localhost:3000",
- "webRoot": "${workspaceFolder}/gui"
- },
- {
- "name": "Python: API Flask (0.11.x or later)",
- "type": "python",
- "request": "launch",
- "module": "flask",
- "env": {
- "FLASK_APP": "nomad/api.py"
- },
- "args": [
- "run",
- "--port", "8000",
- "--no-debugger",
- "--no-reload"
- ]
- },
- {
- "name": "Python: worker nomad.processing",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/celery",
- "args": [
- "worker", "-l" , "debug", "-A", "nomad.processing"
- ]
- },
- {
- "name": "Python: some test",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_api.py::TestRepo::test_search[2-user-other_test_user]"
- ]
- },
- {
- "name": "Python: crystal normalizer test",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/crystal-tests/data/parsers/crystal/si.out]"
- ]
- },
- {
- "name": "Python: cp2k normalizer test",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/cp2k-tests/data/parsers/cp2k/si_bulk8.out]"
- ]
- },
- {
- "name": "Python: cpmd normalizer test",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/cpmd-tests/data/parsers/cpmd/geo_output.out]"
- ]
- },
- {
- "name": "Python: wien2k normalizer test",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/wien2k-tests/data/parsers/wien2k/AlN/AlN_ZB.scf]"
- ]
- },
- {
- "name": "Python: gaussian normalizer test",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/gaussian-tests/data/parsers/gaussian/Al.out]"
- ]
- },
- {
- "name": "Python: aniline gaussian normalizer test",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/gaussian-tests/data/parsers/gaussian/al-1.out]"
- ]
- },
- {
- "name": "Python: test_parsing match test",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_parsing.py::test_match"
- ]
- },
- {
- "name": "Python: nwchem normalizer test h2o sp test",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/nwchem-tests/data/parsers/nwchem/sp_output.out]"
- ]
- },
- {
- "name": "Python: Vasp XML test",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/vasp-tests/data/parsers/vasp/vasp.xml]"
- ]
- },
- {
- "name": "Quantum Espresso Normalizer",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/quantumespresso-tests/data/parsers/quantum-espresso/W.out]"
- ]
- },
- {
- "name": "Abinit Normalizer",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/abinit-tests/data/parsers/abinit/Fe.out]"
- ]
- },
- {
- "name": "Castep Normalizer",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/castep-tests/data/parsers/castep/BC2N-Pmm2-Raman.castep]"
- ]
- },
- {
- "name": "DL-Poly Normalizer",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/dl-poly-tests/data/parsers/dl-poly/OUTPUT]"
- ]
- },
- {
- "name": "Lib Atoms Normalizer",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/lib-atoms-tests/data/parsers/lib-atoms/gp.xml]"
- ]
- },
- {
- "name": "Octopus Normalizer",
- "type": "python",
- "request": "launch",
- "cwd": "${workspaceFolder}",
- "program": "${workspaceFolder}/.pyenv/bin/pytest",
- "args": [
- "-sv", "tests/test_normalizing.py::test_normalizer[parsers/octopus-tests/data/parsers/octopus/stdout.txt]"
- ]
- },
- {
- "name": "Python: Current File",
- "type": "python",
- "request": "launch",
- "program": "${file}"
- },
- {
- "name": "Python: Attach",
- "type": "python",
- "request": "attach",
- "localRoot": "${workspaceFolder}",
- "remoteRoot": "${workspaceFolder}",
- "port": 3000,
- "secret": "my_secret",
- "host": "localhost"
- }
- ]
-}
\ No newline at end of file
diff --git a/nomad/client/local.py b/nomad/client/local.py
index 1b77da741b28b78c031eab2e38cea33d5b205feb..5b15ba910d40a8114a4f6bfcbbebf8d679b0b464 100644
--- a/nomad/client/local.py
+++ b/nomad/client/local.py
@@ -26,7 +26,7 @@ from nomad.files import ArchiveBasedStagingUploadFiles
from nomad.parsing import parser_dict, LocalBackend, match_parser
from nomad.normalizing import normalizers
-from .main import cli, get_nomad_url
+from .main import cli
class CalcProcReproduction:
@@ -67,7 +67,7 @@ class CalcProcReproduction:
# TODO currently only downloads mainfile
self.logger.info('Downloading calc.', mainfile=self.mainfile)
token = client.auth.get_user().response().result.token
- req = requests.get('%s/raw/%s/%s' % (get_nomad_url(), self.upload_id, os.path.dirname(self.mainfile)) + '/*', stream=True, headers={'X-Token': token})
+ req = requests.get('%s/raw/%s/%s' % (config.client.url, self.upload_id, os.path.dirname(self.mainfile)) + '/*', stream=True, headers={'X-Token': token})
with open(local_path, 'wb') as f:
for chunk in req.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE):
f.write(chunk)
@@ -166,7 +166,7 @@ class CalcProcReproduction:
def local(archive_id, show_backend=False, show_metadata=False, **kwargs):
print(kwargs)
utils.configure_logging()
- utils.get_logger(__name__).info('Using %s' % get_nomad_url())
+ utils.get_logger(__name__).info('Using %s' % config.client.url)
with CalcProcReproduction(archive_id, **kwargs) as local:
backend = local.parse()
local.normalize_all(parser_backend=backend)
diff --git a/nomad/client/main.py b/nomad/client/main.py
index 4fa5a07a90f2c86820cca6d54d377fc57f33b69e..e9637c67823903128db304fe4d0c9adecdf3433f 100644
--- a/nomad/client/main.py
+++ b/nomad/client/main.py
@@ -24,16 +24,6 @@ from urllib.parse import urlparse
from nomad import config, utils, infrastructure
-_default_url = 'http://%s:%d/%s' % (config.services.api_host, config.services.api_port, config.services.api_base_path.strip('/'))
-_nomad_url = os.environ.get('NOMAD_URL', _default_url)
-_user = os.environ.get('NOMAD_USER', 'leonard.hofstadter@nomad-fairdi.tests.de')
-_pw = os.environ.get('NOMAD_PASSWORD', 'password')
-
-
-def get_nomad_url():
- return _nomad_url
-
-
def create_client():
return _create_client()
@@ -42,15 +32,15 @@ def _create_client(*args, **kwargs):
return __create_client(*args, **kwargs)
-def __create_client(user: str = _user, password: str = _pw):
+def __create_client(user: str = config.client.user, password: str = config.client.password):
""" A factory method to create the client. """
- host = urlparse(_nomad_url).netloc.split(':')[0]
+ host = urlparse(config.client.url).netloc.split(':')[0]
http_client = RequestsClient()
if user is not None:
http_client.set_basic_auth(host, user, password)
client = SwaggerClient.from_url(
- '%s/swagger.json' % _nomad_url,
+ '%s/swagger.json' % config.client.url,
http_client=http_client)
utils.get_logger(__name__).info('created bravado client', user=user)
@@ -65,15 +55,15 @@ def handle_common_errors(func):
except requests.exceptions.ConnectionError:
click.echo(
'\nCould not connect to nomad at %s. '
- 'Check connection and url.' % _nomad_url)
+ 'Check connection and url.' % config.client.url)
sys.exit(0)
return wrapper
@click.group()
-@click.option('-n', '--url', default=_nomad_url, help='The URL where nomad is running "%s".' % _nomad_url)
-@click.option('-u', '--user', default=None, help='the user name to login, default no login.')
-@click.option('-w', '--password', default=_pw, help='the password use to login.')
+@click.option('-n', '--url', default=config.client.url, help='The URL where nomad is running, default is "%s".' % config.client.url)
+@click.option('-u', '--user', default=None, help='the user name to login, default is "%s" login.' % config.client.user)
+@click.option('-w', '--password', default=config.client.password, help='the password used to login.')
@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
@click.option('--debug', help='sets log level to debug', is_flag=True)
def cli(url: str, verbose: bool, debug: bool, user: str, password: str):
@@ -92,8 +82,7 @@ def cli(url: str, verbose: bool, debug: bool, user: str, password: str):
logger.info('Used nomad is %s' % url)
logger.info('Used user is %s' % user)
- global _nomad_url
- _nomad_url = url
+ config.client.url = url
global _create_client
diff --git a/nomad/client/upload.py b/nomad/client/upload.py
index fae44a59332c4c739ea05c8eb28b8e5f4e5c2c95..1b86a37fd846e2a1be22d5ae16a364e80013e8ad 100644
--- a/nomad/client/upload.py
+++ b/nomad/client/upload.py
@@ -19,17 +19,16 @@ import click
import urllib.parse
import requests
-from nomad import utils
+from nomad import utils, config
from nomad.processing import FAILURE, SUCCESS
from .main import cli, create_client
-from nomad.client import main
def stream_upload_with_client(client, stream, name=None):
user = client.auth.get_user().response().result
token = user.token
- url = main._nomad_url + '/uploads/'
+ url = config.client.url + '/uploads/'
if name is not None:
url += '?name=%s' % urllib.parse.quote(name)
diff --git a/nomad/config.py b/nomad/config.py
index 90b19b6405b8afd5f88066c284d1546c207918b5..336830b472bbe84ca3cc4304ffe13a8f582fd590 100644
--- a/nomad/config.py
+++ b/nomad/config.py
@@ -12,68 +12,132 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""
-This module is used to store all configuration values. It makes use of
-*namedtuples* to create key sensitive configuration objects.
-"""
-
-import os
import logging
-from collections import namedtuple
+import os
+import os.path
+import yaml
import warnings
+from nomad import utils
+
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
+class NomadConfig(utils.POPO):
+ pass
+
+
+# class ConfigProperty:
+# def __init__(
+# self, name: str, default_value: Union[int, str, bool], help: str = None,
+# env_var: str = None) -> None:
+# self.name = name
+# self.default_value = default_value,
+# self.help = help
+# self.env_var = env_var
+
CELERY_WORKER_ROUTING = 'worker'
CELERY_QUEUE_ROUTING = 'queue'
-CeleryConfig = namedtuple('Celery', [
- 'broker_url', 'max_memory', 'timeout', 'acks_late', 'routing'])
-"""
-Used to configure the RabbitMQ for celery.
+rabbitmq = NomadConfig(
+ host='localhost',
+ user='rabbitmq',
+ password='rabbitmq'
+)
-Arguments:
- broker_url: The rabbitmq broker URL
- max_memory: Max worker memory
- timeout: Task timeout
- acks_late: Use celery acks_late if set
- routing: Set to ``queue`` for routing via upload, calc queues. Processing tasks for one
- upload might be routed to different worker and all staging files must be accessible
- for all workers via distributed fs. Set to ``worker`` to route all tasks related
- to the same upload to the same worker.
-"""
-FSConfig = namedtuple('FSConfig', ['tmp', 'staging', 'public', 'prefix_size'])
-""" Used to configure file stystem access. """
+def rabbitmq_url():
+ return 'pyamqp://%s:%s@%s//' % (rabbitmq.user, rabbitmq.password, rabbitmq.host)
-RepositoryDBConfig = namedtuple('RepositoryDBConfig', ['host', 'port', 'dbname', 'user', 'password'])
-""" Used to configure access to NOMAD-coe repository db. """
-ElasticConfig = namedtuple('ElasticConfig', ['host', 'port', 'index_name'])
-""" Used to configure elastic search. """
+celery = NomadConfig(
+ max_memory=64e6, # 64 GB
+ timeout=1800, # 1/2 h
+ acks_late=True,
+ routing=CELERY_QUEUE_ROUTING
+)
-MongoConfig = namedtuple('MongoConfig', ['host', 'port', 'db_name'])
-""" Used to configure mongo db. """
+fs = NomadConfig(
+ tmp='.volumes/fs/tmp',
+ staging='.volumes/fs/staging',
+ public='.volumes/fs/public',
+ prefix_size=2
+)
-LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port', 'level'])
-""" Used to configure and enable/disable the ELK based centralized logging. """
+elastic = NomadConfig(
+ host='localhost',
+ port=9200,
+ index_name='nomad_fairdi_calcs'
+)
-NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_host', 'api_port', 'api_base_path', 'api_secret', 'admin_password', 'upload_url', 'disable_reset'])
-""" Used to configure nomad services: worker, handler, api """
+repository_db = NomadConfig(
+ host='localhost',
+ port=5432,
+ dbname='nomad_fairdi_repo_db',
+ user='postgres',
+ password='nomad'
+)
-MailConfig = namedtuple('MailConfig', ['host', 'port', 'user', 'password', 'from_address'])
-""" Used to configure how nomad can send email """
+mongo = NomadConfig(
+ host='localhost',
+ port=27017,
+ db_name='nomad_fairdi'
+)
-NormalizeConfig = namedtuple('NormalizeConfig', ['all_systems'])
-""" Used to configure the normalizers """
+logstash = NomadConfig(
+ enabled=True,
+ host='localhost',
+ tcp_port='5000',
+ level=logging.DEBUG
+)
-rabbit_host = os.environ.get('NOMAD_RABBITMQ_HOST', 'localhost')
-rabbit_port = os.environ.get('NOMAD_RABBITMQ_PORT', None)
-rabbit_user = 'rabbitmq'
-rabbit_password = 'rabbitmq'
-rabbit_url = 'pyamqp://%s:%s@%s//' % (rabbit_user, rabbit_password, rabbit_host)
+services = NomadConfig(
+ api_host='localhost',
+ api_port=8000,
+ api_base_path='/nomad/api',
+ api_secret='defaultApiSecret',
+ admin_password='password',
+ disable_reset=True
+)
+
+
+def upload_url():
+ return 'http://%s:%s/%s/uploads' % (services.api_host, services.api_port, services.api_base_path[:-3])
+
+
+migration_source_db = NomadConfig(
+ host='db-repository.nomad.esc',
+ port=5432,
+ dbname='nomad_prod',
+ user='nomadlab',
+ password='*'
+)
+
+mail = NomadConfig(
+ enabled=False,
+ with_login=False,
+ host='',
+ port=8995,
+ user='',
+ password='',
+ from_address='webmaster@nomad-coe.eu'
+)
+
+normalize = NomadConfig(
+ all_systems=False
+)
+
+client = NomadConfig(
+ user='leonard.hofstadter@nomad-fairdi.tests.de',
+ password='password',
+ url='http://localhost:8000/nomad/api'
+)
+
+console_log_level = logging.WARNING
+service = 'unknown nomad service'
+release = 'devel'
+auxfile_cutoff = 30
def get_loglevel_from_env(key, default_level=logging.INFO):
@@ -84,78 +148,113 @@ def get_loglevel_from_env(key, default_level=logging.INFO):
try:
return int(plain_value)
except ValueError:
- return getattr(logging, plain_value, default_level)
+ return getattr(logging, plain_value)
-celery = CeleryConfig(
- broker_url=rabbit_url,
- max_memory=int(os.environ.get('NOMAD_CELERY_MAXMEMORY', 64e6)), # 64 GB
- timeout=int(os.environ.get('NOMAD_CELERY_TIMEOUT', 1800)), # 1/2h
- acks_late=bool(os.environ.get('NOMAD_CELERY_ACKS_LATE', True)),
- routing=os.environ.get('NOMAD_CELERY_ROUTING', CELERY_QUEUE_ROUTING)
-)
-fs = FSConfig(
- tmp=os.environ.get('NOMAD_FILES_TMP_DIR', '.volumes/fs/tmp'),
- staging=os.environ.get('NOMAD_FILES_STAGING_DIR', '.volumes/fs/staging'),
- public=os.environ.get('NOMAD_FILES_PUBLIC_DIR', '.volumes/fs/public'),
- prefix_size=int(os.environ.get('NOMAD_FILES_PREFIX_SIZE', 2))
-)
-elastic = ElasticConfig(
- host=os.environ.get('NOMAD_ELASTIC_HOST', 'localhost'),
- port=int(os.environ.get('NOMAD_ELASTIC_PORT', 9200)),
- index_name=os.environ.get('NOMAD_ELASTIC_INDEX_NAME', 'nomad_fairdi_calcs')
-)
-repository_db = RepositoryDBConfig(
- host=os.environ.get('NOMAD_COE_REPO_DB_HOST', 'localhost'),
- port=int(os.environ.get('NOMAD_COE_REPO_DB_PORT', 5432)),
- dbname=os.environ.get('NOMAD_COE_REPO_DB_NAME', 'nomad_fairdi_repo_db'),
- user=os.environ.get('NOMAD_COE_REPO_DB_USER', 'postgres'),
- password=os.environ.get('NOMAD_COE_REPO_PASSWORD', 'nomad')
-)
-mongo = MongoConfig(
- host=os.environ.get('NOMAD_MONGO_HOST', 'localhost'),
- port=int(os.environ.get('NOMAD_MONGO_PORT', 27017)),
- db_name=os.environ.get('NOMAD_MONGO_DB_NAME', 'nomad_fairdi')
-)
-logstash = LogstashConfig(
- enabled=True,
- host=os.environ.get('NOMAD_LOGSTASH_HOST', 'localhost'),
- tcp_port=int(os.environ.get('NOMAD_LOGSTASH_TCPPORT', '5000')),
- level=get_loglevel_from_env('NOMAD_LOGSTASH_LEVEL', default_level=logging.DEBUG)
-)
-services = NomadServicesConfig(
- api_host=os.environ.get('NOMAD_API_HOST', 'localhost'),
- api_port=int(os.environ.get('NOMAD_API_PORT', 8000)),
- api_base_path=os.environ.get('NOMAD_API_BASE_PATH', '/nomad/api'),
- api_secret=os.environ.get('NOMAD_API_SECRET', 'defaultApiSecret'),
- admin_password=os.environ.get('NOMAD_API_ADMIN_PASSWORD', 'password'),
- upload_url=os.environ.get('NOMAD_UPLOAD_URL', 'http://localhost/nomad/uploads'),
- disable_reset=os.environ.get('NOMAD_API_DISABLE_RESET', 'false') == 'true'
-)
-migration_source_db = RepositoryDBConfig(
- host=os.environ.get('NOMAD_MIGRATION_SOURCE_DB_HOST', 'db-repository.nomad.esc'),
- port=int(os.environ.get('NOMAD_MIGRATION_SOURCE_DB_PORT', 5432)),
- dbname=os.environ.get('NOMAD_MIGRATION_SOURCE_DB_NAME', 'nomad_prod'),
- user=os.environ.get('NOMAD_MIGRATION_SOURCE_USER', 'nomadlab'),
- password=os.environ.get('NOMAD_MIGRATION_SOURCE_PASSWORD', '*')
-)
-mail = MailConfig(
- host=os.environ.get('NOMAD_SMTP_HOST', ''), # empty or None host disables email
- port=int(os.environ.get('NOMAD_SMTP_PORT', 8995)),
- user=os.environ.get('NOMAD_SMTP_USER', None),
- password=os.environ.get('NOMAD_SMTP_PASSWORD', None),
- from_address=os.environ.get('NOMAD_MAIL_FROM', 'webmaster@nomad-coe.eu')
-)
-normalize = NormalizeConfig(
- all_systems=False
-)
+transformations = {
+ 'console_log_level': get_loglevel_from_env,
+ 'logstash_level': get_loglevel_from_env
+}
-console_log_level = get_loglevel_from_env('NOMAD_CONSOLE_LOGLEVEL', default_level=logging.WARNING)
-service = os.environ.get('NOMAD_SERVICE', 'unknown nomad service')
-release = os.environ.get('NOMAD_RELEASE', 'devel')
-auxfile_cutoff = 30
-"""
-Number of max auxfiles. More auxfiles means no auxfiles, but probably a directory of many
-mainfiles.
-"""
+# use std python logger, since logging is not configured while loading configuration
+logger = logging.getLogger(__name__)
+
+
+def apply(config, key, value) -> None:
+ """
+ Changes the config according to given key and value. The keys are interpreted as paths
+ to config values with ``_`` as a separator. E.g. ``fs_staging`` leading to
+ ``config.fs.staging``
+ """
+
+ path = list(reversed(key.split('_')))
+ child_segment = None
+ current_value = None
+ child_config = config
+ child_key = None
+
+ try:
+ while len(path) > 0:
+ if child_segment is None:
+ child_segment = path.pop()
+ else:
+ child_segment += '_' + path.pop()
+
+ if child_segment in child_config:
+ current_value = child_config[child_segment]
+
+ if current_value is None:
+ if len(path) == 0:
+ raise KeyError
+ if isinstance(current_value, NomadConfig):
+ child_config = current_value
+ child_segment = None
+ else:
+ if len(path) > 0:
+ raise KeyError()
+ else:
+ child_key = child_segment
+ break
+
+ if child_key is None or current_value is None:
+ raise KeyError()
+ except KeyError:
+ logger.error('config key %s does not exist' % key)
+
+ if not isinstance(value, type(current_value)):
+ try:
+ value = transformations.get(key, type(current_value))(value)
+ except Exception as e:
+ logger.error(
+ 'config key %s value %s has wrong type: %s' % (key, str(value), str(e)))
+
+ child_config[child_key] = value
+
+
+def load_config(config_file: str = os.environ.get('NOMAD_CONFIG', 'nomad.yml')) -> None:
+ # load yml and override defaults
+ if os.path.exists(config_file):
+ with open(config_file, 'r') as stream:
+ try:
+ config_data = yaml.load(stream)
+ except yaml.YAMLError as e:
+ logger.error('cannot read nomad config', exc_info=e)
+
+ def adapt(config, new_config, child_key=None):
+ for key, value in new_config.items():
+ if key in config:
+ if child_key is None:
+ qualified_key = key
+ else:
+ qualified_key = '%s_%s' % (child_key, key)
+
+ current_value = config[key]
+ if isinstance(value, dict) and isinstance(current_value, NomadConfig):
+ adapt(current_value, value, qualified_key)
+ else:
+ if not isinstance(value, type(current_value)):
+ try:
+ value = transformations.get(qualified_key, type(current_value))(value)
+ except Exception as e:
+ logger.error(
+ 'config key %s value %s has wrong type: %s' % (key, str(value), str(e)))
+ else:
+ config[key] = value
+ else:
+ logger.error('config key %s does not exist' % key)
+
+ adapt(globals(), config_data)
+
+ # load env and override yml and defaults
+ kwargs = {
+ key[len('NOMAD_'):].lower(): value
+ for key, value in os.environ.items()
+ if key.startswith('NOMAD_')
+ }
+ config = globals()
+ for key, value in kwargs.items():
+ apply(config, key, value)
+
+
+load_config()
diff --git a/nomad/infrastructure.py b/nomad/infrastructure.py
index 182bc9d72d0da25d782c4717435834706678f669..b6177dd24dda755656cfd40d4d0680e736ae51b4 100644
--- a/nomad/infrastructure.py
+++ b/nomad/infrastructure.py
@@ -180,7 +180,7 @@ def sqlalchemy_repository_db(exists: bool = False, readonly: bool = True, **kwar
def no_flush():
pass
- params = config.repository_db._asdict()
+ params = dict(**config.repository_db)
params.update(**kwargs)
url = 'postgresql://%s:%s@%s:%d/%s' % utils.to_tuple(params, 'user', 'password', 'host', 'port', 'dbname')
# We tried to set a very high isolation level, to prevent conflicts between transactions on the
@@ -311,15 +311,12 @@ def remove():
@contextmanager
def repository_db_connection(dbname=None, with_trans=True):
""" Contextmanager for a psycopg2 session for the NOMAD-coe repository postgresdb """
- repository_db_dict = config.repository_db._asdict()
- if dbname is not None:
- repository_db_dict.update(dbname=dbname)
conn_str = "host='%s' port=%d dbname='%s' user='%s' password='%s'" % (
- repository_db_dict['host'],
- repository_db_dict['port'],
- repository_db_dict['dbname'],
- repository_db_dict['user'],
- repository_db_dict['password'])
+ config.repository_db.host,
+ config.repository_db.port,
+ config.repository_db.dbname if dbname is None else dbname,
+ config.repository_db.user,
+ config.repository_db.password)
conn = psycopg2.connect(conn_str)
if not with_trans:
@@ -403,7 +400,7 @@ def reset_repository_db_content():
def send_mail(name: str, email: str, message: str, subject: str):
- if config.mail.host is None or config.mail.host.strip() == '':
+ if not config.mail.enabled:
return
logger = utils.get_logger(__name__)
@@ -415,15 +412,15 @@ def send_mail(name: str, email: str, message: str, subject: str):
except Exception as e:
logger.warning('Could use TTS', exc_info=e)
- if config.mail.user is not None:
+ if config.mail.with_login:
try:
- server.login("youremailusername", "password")
+ server.login(config.mail.user, config.mail.password)
except Exception as e:
logger.warning('Could not log into mail server', exc_info=e)
msg = MIMEText(message)
msg['Subject'] = subject
- msg['From'] = 'nomad@fairdi webmaster'
+ msg['From'] = config.mail.from_address
msg['To'] = name
try:
diff --git a/nomad/processing/base.py b/nomad/processing/base.py
index 6fdab84ca940deebb1cfaacf22242754393eff5a..bf7aa238a75c1e61a38cb9688353b5706a3eb66b 100644
--- a/nomad/processing/base.py
+++ b/nomad/processing/base.py
@@ -48,7 +48,7 @@ def setup(**kwargs):
'celery configured with acks_late=%s' % str(config.celery.acks_late))
-app = Celery('nomad.processing', broker=config.celery.broker_url)
+app = Celery('nomad.processing', broker=config.rabbitmq_url())
app.conf.update(worker_hijack_root_logger=False)
app.conf.update(worker_max_memory_per_child=config.celery.max_memory)
if config.celery.routing == config.CELERY_WORKER_ROUTING:
diff --git a/nomad/utils.py b/nomad/utils.py
index b3a1e4cb0689076ff6964ced0094c0a518928c9f..ec0eeb5cb93e99eb372469485e344258175c020f 100644
--- a/nomad/utils.py
+++ b/nomad/utils.py
@@ -48,8 +48,6 @@ import re
from werkzeug.exceptions import HTTPException
import hashlib
-from nomad import config
-
default_hash_len = 28
""" Length of hashes and hash-based ids (e.g. calc, upload) in nomad. """
@@ -166,6 +164,7 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase):
message[key] = value
else:
+ from nomad import config
structlog['nomad.service'] = config.service
structlog['nomad.release'] = config.release
message.update(structlog)
@@ -213,6 +212,7 @@ def add_logstash_handler(logger):
if isinstance(handler, LogstashHandler)), None)
if logstash_handler is None:
+ from nomad import config
logstash_handler = LogstashHandler(
config.logstash.host,
config.logstash.tcp_port, version=1)
@@ -222,6 +222,8 @@ def add_logstash_handler(logger):
def configure_logging():
+ from nomad import config
+
# configure structlog
log_processors = [
StackInfoRenderer(),
@@ -275,6 +277,8 @@ def get_logger(name, **kwargs):
Returns a structlog logger that is already attached with a logstash handler.
Use additional *kwargs* to pre-bind some values to all events.
"""
+ from nomad import config
+
if name.startswith('nomad.'):
name = '.'.join(name.split('.')[:2])
diff --git a/ops/docker-compose/nomad/docker-compose.develk.yml b/ops/docker-compose/nomad/docker-compose.develk.yml
index 9bf20e8a5958962c176b7e9b5f335ada1d0d5f9f..2acc7b9486c2cef1d0e88721e7d05d852e110b25 100644
--- a/ops/docker-compose/nomad/docker-compose.develk.yml
+++ b/ops/docker-compose/nomad/docker-compose.develk.yml
@@ -52,7 +52,7 @@ services:
image: nomad/backend
environment:
NOMAD_LOGSTASH_LEVEL: DEBUG
- NOMAD_CONSOLE_LOGLEVEL: INFO
+ NOMAD_CONSOLE_LOG_LEVEL: INFO
links:
- elk
@@ -62,7 +62,7 @@ services:
image: nomad/backend
environment:
NOMAD_LOGSTASH_LEVEL: DEBUG
- NOMAD_CONSOLE_LOGLEVEL: INFO
+ NOMAD_CONSOLE_LOG_LEVEL: INFO
depends_on:
- worker
ports:
diff --git a/ops/docker-compose/nomad/docker-compose.yml b/ops/docker-compose/nomad/docker-compose.yml
index b050044fcc85de8447c0cb62359571a858a9ce78..ae68974fdc546716edacefff318c1c7f3929372a 100644
--- a/ops/docker-compose/nomad/docker-compose.yml
+++ b/ops/docker-compose/nomad/docker-compose.yml
@@ -19,8 +19,6 @@ x-common-variables: &nomad_backend_env
NOMAD_LOGSTASH_HOST: elk
NOMAD_ELASTIC_HOST: elastic
NOMAD_MONGO_HOST: mongo
- NOMAD_SMTP_HOST: ''
- NOMAD_UPLOAD_URL: ''
services:
# postgres for NOMAD-coe repository API and GUI
@@ -92,11 +90,11 @@ services:
container_name: nomad_api
environment:
<<: *nomad_backend_env
- NOMAD_API_BASE_PATH: /nomad/api
- NOMAD_API_HOST: ${EXTERNAL_HOST}
- NOMAD_API_PORT: ${EXTERNAL_PORT}
+ NOMAD_SERVICES_API_BASE_PATH: /nomad/api
+ NOMAD_SERVICES_API_HOST: ${EXTERNAL_HOST}
+ NOMAD_SERVICES_API_PORT: ${EXTERNAL_PORT}
+ NOMAD_SERVICES_API_SECRET: ${API_SECRET}
NOMAD_SERVICE: nomad_api
- NOMAD_API_SECRET: ${API_SECRET}
links:
- postgres
- rabbitmq
diff --git a/ops/helm/nomad-full/.gitignore b/ops/helm/nomad-full/.gitignore
deleted file mode 100644
index 711a39c541afc04f86b51e268b350d204f48c875..0000000000000000000000000000000000000000
--- a/ops/helm/nomad-full/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-charts/
\ No newline at end of file
diff --git a/ops/helm/nomad-full/.helmignore b/ops/helm/nomad-full/.helmignore
deleted file mode 100644
index 50af0317254197a5a019f4ac2f8ecc223f93f5a7..0000000000000000000000000000000000000000
--- a/ops/helm/nomad-full/.helmignore
+++ /dev/null
@@ -1,22 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/
diff --git a/ops/helm/nomad-full/Chart.yaml b/ops/helm/nomad-full/Chart.yaml
deleted file mode 100644
index 9796ee1645c38db0a0906b5ea7cb91f7c2248bb3..0000000000000000000000000000000000000000
--- a/ops/helm/nomad-full/Chart.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-apiVersion: v1
-appVersion: "1.0"
-description: A Helm chart for Kubernetes that runs the whole nomad including databases.
-name: nomad-full
-version: 0.1.0
diff --git a/ops/helm/nomad-full/requirements.lock b/ops/helm/nomad-full/requirements.lock
deleted file mode 100644
index 9cc3a25afa8d9a24b953f82908e02f772e645911..0000000000000000000000000000000000000000
--- a/ops/helm/nomad-full/requirements.lock
+++ /dev/null
@@ -1,15 +0,0 @@
-dependencies:
-- name: rabbitmq
- repository: https://kubernetes-charts.storage.googleapis.com/
- version: 4.0.1
-- name: mongodb
- repository: https://kubernetes-charts.storage.googleapis.com/
- version: 4.9.1
-- name: elasticsearch
- repository: https://kubernetes-charts.storage.googleapis.com/
- version: 1.15.0
-- name: postgresql
- repository: https://kubernetes-charts.storage.googleapis.com/
- version: 3.1.1
-digest: sha256:c6c65e79414429b8b2ecdba6fa2f13628614a0be3c5fff8947c79270a00386fb
-generated: 2018-12-12T14:02:40.726497+01:00
diff --git a/ops/helm/nomad-full/requirements.yaml b/ops/helm/nomad-full/requirements.yaml
deleted file mode 100644
index a4f11fa4ddc24477738e29a7cf681dc60b854055..0000000000000000000000000000000000000000
--- a/ops/helm/nomad-full/requirements.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-dependencies:
-- name: rabbitmq
- version: "4.0.1"
- repository: "https://kubernetes-charts.storage.googleapis.com/"
-- name: mongodb
- version: "4.9.1"
- repository: "https://kubernetes-charts.storage.googleapis.com/"
-- name: elasticsearch
- version: "1.15.0"
- repository: "https://kubernetes-charts.storage.googleapis.com/"
-- name: postgresql
- version: "3.1.1"
- repository: "https://kubernetes-charts.storage.googleapis.com/"
\ No newline at end of file
diff --git a/ops/helm/nomad-full/templates/_helpers.tpl b/ops/helm/nomad-full/templates/_helpers.tpl
deleted file mode 100644
index 1d5277e45c79510691d3311e97bea01702d3f5bd..0000000000000000000000000000000000000000
--- a/ops/helm/nomad-full/templates/_helpers.tpl
+++ /dev/null
@@ -1,32 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "nomad.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "nomad.fullname" -}}
-{{- if .Values.fullnameOverride -}}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- if contains $name .Release.Name -}}
-{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "nomad.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
diff --git a/ops/helm/nomad-full/templates/api-deployment.yaml b/ops/helm/nomad-full/templates/api-deployment.yaml
deleted file mode 100644
index 82f5a412eaee0cb90ad52cc29b28e582721e8f86..0000000000000000000000000000000000000000
--- a/ops/helm/nomad-full/templates/api-deployment.yaml
+++ /dev/null
@@ -1,49 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: {{ include "nomad.fullname" . }}-api
- labels:
- app.kubernetes.io/name: {{ include "nomad.name" . }}-api
- helm.sh/chart: {{ include "nomad.chart" . }}
- app.kubernetes.io/instance: {{ .Release.Name }}
- app.kubernetes.io/managed-by: {{ .Release.Service }}
-spec:
- replicas: {{ .Values.api.replicas }}
- selector:
- matchLabels:
- app.kubernetes.io/name: {{ include "nomad.name" . }}-api
- app.kubernetes.io/instance: {{ .Release.Name }}
- template:
- metadata:
- labels:
- app.kubernetes.io/name: {{ include "nomad.name" . }}-api
- app.kubernetes.io/instance: {{ .Release.Name }}
- spec:
- containers:
- - name: {{ include "nomad.name" . }}-api
- image: "{{ .Values.images.nomad.name }}:{{ .Values.images.nomad.tag }}"
- volumeMounts:
- - mountPath: /app/.volumes/fs
- name: files-volume
- env:
- - name: NOMAD_SERVICE
- value: "api"
- - name: NOMAD_LOGSTASH_HOST
- value: {{ .Values.logstash.host }}
- - name: NOMAD_LOGSTASH_TCPPORT
- value: {{ .Values.logstash.port }}
- - name: NOMAD_CONSOLE_LOGLEVEL
- value: {{ .Values.api.console_loglevel }}
- - name: NOMAD_LOGSTASH_LEVEL
- value: {{ .Values.api.logstash_loglevel }}
- - name: NOMAD_API_PORT
- value: {{ .Values.api.port }}
- - name: NOMAD_API_SECRET
- value: {{ .Values.api.secret }}
- imagePullSecrets:
- - name: {{ .Values.images.secret }}
- volumes:
- - name: files-volume
- hostPath:
- path: {{ .Values.volumes.files }}
- type: Directory
diff --git a/ops/helm/nomad-full/templates/api-service.yaml b/ops/helm/nomad-full/templates/api-service.yaml
deleted file mode 100644
index 0a7cb7ac73558caeee79043761028fe59de055c1..0000000000000000000000000000000000000000
--- a/ops/helm/nomad-full/templates/api-service.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
- name: {{ include "nomad.fullname" . }}-api
- labels:
- app.kubernetes.io/name: {{ include "nomad.name" . }}-api
- helm.sh/chart: {{ include "nomad.chart" . }}
- app.kubernetes.io/instance: {{ .Release.Name }}
- app.kubernetes.io/managed-by: {{ .Release.Service }}
-spec:
- type: {{ .Values.api.service_type }}
- ports:
- - port: {{ .Values.api.port }}
- targetPort: http
- protocol: TCP
- name: http
- selector:
- app.kubernetes.io/name: {{ include "nomad.name" . }}-api
- app.kubernetes.io/instance: {{ .Release.Name }}
diff --git a/ops/helm/nomad-full/templates/worker-deployment.yaml b/ops/helm/nomad-full/templates/worker-deployment.yaml
deleted file mode 100644
index 6d9a1aaf6bb45d04735b5d077879b1b9f73addb7..0000000000000000000000000000000000000000
--- a/ops/helm/nomad-full/templates/worker-deployment.yaml
+++ /dev/null
@@ -1,45 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: {{ include "nomad.fullname" . }}-worker
- labels:
- app.kubernetes.io/name: {{ include "nomad.name" . }}-worker
- helm.sh/chart: {{ include "nomad.chart" . }}
- app.kubernetes.io/instance: {{ .Release.Name }}
- app.kubernetes.io/managed-by: {{ .Release.Service }}
-spec:
- replicas: {{ .Values.worker.replicas }}
- selector:
- matchLabels:
- app.kubernetes.io/name: {{ include "nomad.name" . }}-worker
- app.kubernetes.io/instance: {{ .Release.Name }}
- template:
- metadata:
- labels:
- app.kubernetes.io/name: {{ include "nomad.name" . }}-worker
- app.kubernetes.io/instance: {{ .Release.Name }}
- spec:
- containers:
- - name: {{ include "nomad.name" . }}-worker
- image: "{{ .Values.images.nomad.name }}:{{ .Values.images.nomad.tag }}"
- volumeMounts:
- - mountPath: /app/.volumes/fs
- name: files-volume
- env:
- - name: NOMAD_SERVICE
- value: "worker"
- - name: NOMAD_LOGSTASH_HOST
- value: {{ .Values.logstash.host }}
- - name: NOMAD_LOGSTASH_TCPPORT
- value: {{ .Values.logstash.port }}
- - name: NOMAD_CONSOLE_LOGLEVEL
- value: {{ .Values.worker.console_loglevel }}
- - name: NOMAD_LOGSTASH_LEVEL
- value: {{ .Values.worker.logstash_loglevel }}
- imagePullSecrets:
- - name: {{ .Values.images.secret }}
- volumes:
- - name: files-volume
- hostPath:
- path: {{ .Values.volumes.files }}
- type: Directory
diff --git a/ops/helm/nomad-full/values.yaml b/ops/helm/nomad-full/values.yaml
deleted file mode 100644
index fafac326be16cb17c94ab85fdd77d92d20e97980..0000000000000000000000000000000000000000
--- a/ops/helm/nomad-full/values.yaml
+++ /dev/null
@@ -1,69 +0,0 @@
-## Default values for nomad@FAIRDI
-
-## Everything concerning the container images to be used
-images:
- ## The kubernetes docker-registry secret that can be used to access the registry
- # with the container image in it.
- # It can be created via:
- # kubectl create secret docker-registry gitlab-mpcdf --docker-server=gitlab-registry.mpcdf.mpg.de --docker-username=<your-user-name > --docker-password=<yourpass> --docker-email=<email>
- secret: gitlab-mpcdf
-
- ## The nomad image with all nomad relavant python code. Used by api and worker service.
- nomad:
- ## The docker container image name without tag
- name: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair
- ## The docker container image tag
- tag: latest
- pullPolicy: IfNotPresent
-
-## Everthing concerning the nomad api
-api:
- replicas: 1
- port: 8000
- console_loglevel: INFO
- logstash_loglevel: INFO
- service_type: ClusterIP
- ## Secret used as cryptographic seed
- secret: "defaultApiSecret"
-
-## Everthing concerning the nomad worker
-worker:
- replicas: 2
- console_loglevel: INFO
- logstash_loglevel: INFO
-
-rabbitmq:
- image.pullSecrets: nil
- rbacEnabled: false
- rabbitmq:
- username: rabbitmq
- password: rabbitmq
- erlangCookie: SWQOKODSQALRPCLNMEQG
- persistence.enabled: false
- securityContext.enabled: false
-
-mongodb:
- image.pullSecrets: nil
- usePassword: false
- persistence.enabled: true
- securityContext.enabled: false
-
-elasticsearch:
- client.replicas: 2
- master.replicas: 1
- data.replicas: 1
- cluster.env: {MINIMUM_MASTER_NODES=""1"}
-
-postgresql:
- postgresqlUsername: "postgres"
- postgresqlPassword: "nomad"
- postgresqlDatabase: "nomad"
- securityContext.enabled: false
-
-logstash:
- port: 5000
- host: "enc-preprocessing-nomad.esc"
-
-## Everthing concerning the data that is used by the service
-volumes:
- files: /nomad/nomadlab/nomad-FAIRDI/files
diff --git a/ops/helm/nomad/templates/api-deployment.yaml b/ops/helm/nomad/templates/api-deployment.yaml
index 64f0d1b08f98d673358cc6b8e8a4fbd55c6f5be1..1640e3f468b13bf7ea9131d1f0d3df8bca100dd2 100644
--- a/ops/helm/nomad/templates/api-deployment.yaml
+++ b/ops/helm/nomad/templates/api-deployment.yaml
@@ -23,63 +23,21 @@ spec:
- name: {{ include "nomad.name" . }}-api
image: "{{ .Values.images.nomad.name }}:{{ .Values.images.nomad.tag }}"
volumeMounts:
+ - mountPath: /app
+ name: nomad-conf
- mountPath: /app/.volumes/fs/public
name: public-volume
- mountPath: /app/.volumes/fs/staging
name: staging-volume
- mountPath: /nomad
name: nomad-volume
- env:
- - name: NOMAD_FILES_TMP_DIR
- value: "{{ .Values.volumes.tmp }}"
+ env:
- name: NOMAD_SERVICE
value: "api"
- - name: NOMAD_RELEASE
- value: "{{ .Release.Name }}"
- - name: NOMAD_LOGSTASH_HOST
- value: "{{ .Values.logstash.host }}"
- - name: NOMAD_LOGSTASH_TCPPORT
- value: "{{ .Values.logstash.port }}"
- name: NOMAD_CONSOLE_LOGLEVEL
value: "{{ .Values.api.console_loglevel }}"
- name: NOMAD_LOGSTASH_LEVEL
value: "{{ .Values.api.logstash_loglevel }}"
- - name: NOMAD_API_HOST
- value: "{{ .Values.proxy.external.host }}"
- - name: NOMAD_API_PORT
- value: "{{ .Values.proxy.external.port }}"
- - name: NOMAD_API_BASE_PATH
- value: "{{ .Values.proxy.external.path }}/api"
- - name: NOMAD_API_SECRET
- value: "{{ .Values.api.secret }}"
- - name: NOMAD_API_ADMIN_PASSWORD
- value: "{{ .Values.api.adminPassword }}"
- - name: NOMAD_API_DISABLE_RESET
- value: "{{ .Values.api.disableReset }}"
- - name: NOMAD_RABBITMQ_HOST
- value: "{{ .Release.Name }}-rabbitmq"
- - name: NOMAD_ELASTIC_HOST
- value: "{{ .Values.elastic.host }}"
- - name: NOMAD_ELASTIC_PORT
- value: "{{ .Values.elastic.port }}"
- - name: NOMAD_ELASTIC_INDEX_NAME
- value: "{{ .Values.dbname }}"
- - name: NOMAD_MONGO_HOST
- value: "{{ .Values.mongo.host }}"
- - name: NOMAD_MONGO_PORT
- value: "{{ .Values.mongo.port }}"
- - name: NOMAD_MONGO_DB_NAME
- value: "{{ .Values.dbname }}"
- - name: NOMAD_COE_REPO_DB_HOST
- value: "{{ .Values.postgres.host }}"
- - name: NOMAD_COE_REPO_DB_PORT
- value: "{{ .Values.postgres.port }}"
- - name: NOMAD_COE_REPO_DB_NAME
- value: "{{ .Values.dbname }}"
- - name: NOMAD_CELERY_ROUTING
- value: "{{ .Values.worker.routing }}"
- - name: NOMAD_FILES_PREFIX_SIZE
- value: "{{ .Values.volumes.prefixSize }}"
command: ["python", "-m", "gunicorn.app.wsgiapp", "--timeout", "3600", "--log-config", "ops/gunicorn.log.conf", "-w", "{{ .Values.api.worker }}", "-b 0.0.0.0:8000", "nomad.api:app"]
livenessProbe:
httpGet:
@@ -99,6 +57,12 @@ spec:
- name: {{ .Values.images.secret }}
imagePullPolicy: always
volumes:
+ - name: nomad-conf
+ configMap:
+ name: {{ include "nomad.fullname" . }}-configmap
+ items:
+ - key: nomad.yml
+ path: nomad.yml
- name: public-volume
hostPath:
path: {{ .Values.volumes.public }}
diff --git a/ops/helm/nomad/templates/nomad-configmap.yml b/ops/helm/nomad/templates/nomad-configmap.yml
new file mode 100644
index 0000000000000000000000000000000000000000..afb07e1da34df2d0f1b463e44b3b42ddd78be9c9
--- /dev/null
+++ b/ops/helm/nomad/templates/nomad-configmap.yml
@@ -0,0 +1,51 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "nomad.fullname" . }}-configmap
+ labels:
+ app.kubernetes.io/name: {{ include "nomad.name" . }}-configmap
+ helm.sh/chart: {{ include "nomad.chart" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+ nomad.yml: /
+ service: api
+ api_host:
+ release: "{{ .Release.Name }}"
+ fs:
+ tmp: "{{ .Values.volumes.tmp }}"
+ prefix_size: {{ .Values.volumes.prefixSize }}
+ logstash:
+ enabled: true
+ host: "{{ .Values.logstash.host }}"
+ tcpport: "{{ .Values.logstash.port }}"
+ level: info
+ console_log_level: debug
+ services:
+ api_host: "{{ .Values.proxy.external.host }}"
+ api_port: "{{ .Values.proxy.external.port }}"
+ api_base_path: "{{ .Values.proxy.external.path }}/api"
+ api_secret: "{{ .Values.api.secret }}"
+ admin_password: "{{ .Values.api.adminPassword }}"
+ disable_reset: {{ .Values.api.disableReset }}
+ rabbitmq:
+ host: "{{ .Release.Name }}-rabbitmq"
+ elastic:
+ host: "{{ .Values.elastic.host }}"
+ port: {{ .Values.elastic.port }}
+ index_name: "{{ .Values.dbname }}"
+ mongo:
+ host: "{{ .Values.mongo.host }}"
+ port: {{ .Values.mongo.port }}
+ db_name: "{{ .Values.dbname }}"
+ repository_db:
+ host: "{{ .Values.postgres.host }}"
+ port: {{ .Values.postgres.port }}
+ dbname: "{{ .Values.dbname }}"
+ mail:
+ host: "{{ .Values.mail.host }}"
+ port: {{ .Values.mail.port }}
+ user: "{{ .Values.mail.user }}"
+ password: "{{ .Values.mail.password }}"
+ from_address: "{{ .Values.mail.from }}"
+ celery_routing: "{{ .Values.worker.routing }}"
\ No newline at end of file
diff --git a/ops/helm/nomad/templates/worker-deployment.yaml b/ops/helm/nomad/templates/worker-deployment.yaml
index c24b5047a146813797937da8d71b280ca7735ab6..1c47b9733db16cc9e89d8e58765ab687f3ae2a1d 100644
--- a/ops/helm/nomad/templates/worker-deployment.yaml
+++ b/ops/helm/nomad/templates/worker-deployment.yaml
@@ -28,6 +28,8 @@ spec:
requests:
memory: "{{ .Values.worker.memrequest }}Gi"
volumeMounts:
+ - mountPath: /app
+ name: nomad-conf
- mountPath: /app/.volumes/fs/public
name: public-volume
- mountPath: /app/.volumes/fs/staging
@@ -35,58 +37,12 @@ spec:
- mountPath: /nomad
name: nomad-volume
env:
- - name: NOMAD_FILES_TMP_DIR
- value: "{{ .Values.volumes.tmp }}"
- name: NOMAD_SERVICE
value: "worker"
- - name: NOMAD_RELEASE
- value: "{{ .Release.Name }}"
- - name: NOMAD_LOGSTASH_HOST
- value: "{{ .Values.logstash.host }}"
- - name: NOMAD_LOGSTASH_TCPPORT
- value: "{{ .Values.logstash.port }}"
- name: NOMAD_CONSOLE_LOGLEVEL
value: "{{ .Values.worker.console_loglevel }}"
- name: NOMAD_LOGSTASH_LEVEL
value: "{{ .Values.worker.logstash_loglevel }}"
- - name: NOMAD_RABBITMQ_HOST
- value: "{{ .Release.Name }}-rabbitmq"
- - name: NOMAD_ELASTIC_HOST
- value: "{{ .Values.elastic.host }}"
- - name: NOMAD_ELASTIC_PORT
- value: "{{ .Values.elastic.port }}"
- - name: NOMAD_ELASTIC_INDEX_NAME
- value: "{{ .Values.dbname }}"
- - name: NOMAD_MONGO_HOST
- value: "{{ .Values.mongo.host }}"
- - name: NOMAD_MONGO_PORT
- value: "{{ .Values.mongo.port }}"
- - name: NOMAD_MONGO_DB_NAME
- value: "{{ .Values.dbname }}"
- - name: NOMAD_COE_REPO_DB_HOST
- value: "{{ .Values.postgres.host }}"
- - name: NOMAD_COE_REPO_DB_PORT
- value: "{{ .Values.postgres.port }}"
- - name: NOMAD_COE_REPO_DB_NAME
- value: "{{ .Values.dbname }}"
- - name: NOMAD_UPLOAD_URL
- value: "{{ .Values.uploadurl }}"
- - name: NOMAD_SMTP_HOST
- value: "{{ .Values.mail.host }}"
- - name: NOMAD_SMTP_PORT
- value: "{{ .Values.mail.port }}"
- - name: NOMAD_SMTP_USER
- value: "{{ .Values.mail.user }}"
- - name: NOMAD_SMTP_PASSWORD
- value: "{{ .Values.mail.password }}"
- - name: NOMAD_MAIL_FROM
- value: "{{ .Values.mail.from }}"
- - name: NOMAD_CONSOLE_LOGLEVEL
- value: "ERROR"
- - name: NOMAD_CELERY_ROUTING
- value: "{{ .Values.worker.routing }}"
- - name: NOMAD_FILES_PREFIX_SIZE
- value: "{{ .Values.volumes.prefixSize }}"
command: ["python", "-m", "celery", "worker", "-A", "nomad.processing", "-Q", "celery,calcs,uploads"]
livenessProbe:
exec:
@@ -110,6 +66,12 @@ spec:
- name: {{ .Values.images.secret }}
imagePullPolicy: always
volumes:
+ - name: nomad-conf
+ configMap:
+ name: {{ include "nomad.fullname" . }}-configmap
+ items:
+ - key: nomad.yml
+ path: nomad.yml
- name: public-volume
hostPath:
path: {{ .Values.volumes.public }}
diff --git a/ops/scripts/migration.env.sh b/ops/scripts/migration.env.sh
new file mode 100644
index 0000000000000000000000000000000000000000..c33bf76957a9edea6ebebf41824b89018d16f6a8
--- /dev/null
+++ b/ops/scripts/migration.env.sh
@@ -0,0 +1,4 @@
+export NOMAD_CLIENT_URL=http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api
+export NOMAD_CLIENT_USER=admin
+export NOMAD_FS_TMP=/nomad/mscheidg/migration_tmp
+export NOMAD_LOGSTASH_TCP_PORT=15000
diff --git a/ops/scripts/migration_client.env.sh b/ops/scripts/migration_client.env.sh
deleted file mode 100644
index 54cdb85d9f9944b1ddd1c110f8a19a734175b4a5..0000000000000000000000000000000000000000
--- a/ops/scripts/migration_client.env.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-export NOMAD_URL=http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api
-export NOMAD_USER=admin
-export NOMAD_FILES_NOMAD_TMP_DIR=/nomad/mscheidg/migration_tmp
-export NOMAD_LOGSTASH_TCPPORT=15000
diff --git a/ops/scripts/migration_env.sh b/ops/scripts/migration_env.sh
deleted file mode 100644
index 68546852928433eaa987268520e850c58075401e..0000000000000000000000000000000000000000
--- a/ops/scripts/migration_env.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-export NOMAD_URL=http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api
-export NOMAD_USER=admin
-export NOMAD_FILES_NOMAD_TMP_DIR=/nomad/mscheidg/migration_tmp
-export NOMAD_LOGSTASH_TCPPORT=15000
-
-export NOMAD_FILES_TMP_DIR=/scratch/nomad-fair/local/tmp
-export NOMAD_FILES_OBJECT_DIR=/scratch/nomad-fair/local/objects
-export NOMAD_FILES_NOMAD_TMP=/nomad/mscheidg/migration_tmp
-export NOMAD_ELASTIC_PORT=19200
-export NOMAD_ELASTIC_INDEX_NAME=fairdi_nomad_local
-export NOMAD_MONGO_PORT=37017
-export NOMAD_MONGO_DB_NAME=fairdi_nomad_local
-export NOMAD_COE_REPO_DB_NAME=fairdi_nomad_local
diff --git a/tests/__init__.py b/tests/__init__.py
index 482bfbc29ce9e523dcf01ee52fceee554466750e..e48f987ae9f076668b44484fd972c9599a585b40 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -30,11 +30,7 @@ from nomad import config
# changing them afterwards does not change anything anymore.
# For convinience we test the api without path prefix.
-services_config = config.services._asdict()
-services_config.update(api_base_path='')
-config.services = config.NomadServicesConfig(**services_config)
+config.services.api_base_path = ''
# We use a mocked in memory mongo version.
-mongo_config = config.mongo._asdict()
-mongo_config.update(host='mongomock://localhost')
-config.mongo = config.MongoConfig(**mongo_config)
+config.mongo.host = 'mongomock://localhost'
diff --git a/tests/conftest.py b/tests/conftest.py
index 05ba2b02c654083a713b65a0faaaf54945b64c26..b0667d4d0ad8705574460cb34587d96d7b64c843 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -50,16 +50,17 @@ def monkeysession(request):
@pytest.fixture(scope='session', autouse=True)
def nomad_logging():
- config.logstash = config.logstash._replace(enabled=False)
+ config.logstash.enabled = False
config.console_log_level = test_log_level
infrastructure.setup_logging()
@pytest.fixture(scope='session', autouse=True)
-def raw_files_infra(monkeysession):
- monkeysession.setattr('nomad.config.fs', config.FSConfig(
- tmp='.volumes/test_fs/tmp', staging='.volumes/test_fs/staging',
- public='.volumes/test_fs/public', prefix_size=2))
+def raw_files_infra():
+ config.fs.tmp = '.volumes/test_fs/tmp'
+ config.fs.staging = '.volumes/test_fs/staging'
+ config.fs.public = '.volumes/test_fs/public'
+ config.fs.prefix_size = 2
@pytest.fixture(scope='function')
@@ -80,7 +81,7 @@ def raw_files(raw_files_infra):
@pytest.fixture(scope='function')
-def client(monkeysession, mongo):
+def client(mongo):
api.app.config['TESTING'] = True
client = api.app.test_client()
@@ -95,7 +96,7 @@ def celery_includes():
@pytest.fixture(scope='session')
def celery_config():
return {
- 'broker_url': config.celery.broker_url
+ 'broker_url': config.rabbitmq_url()
}
@@ -155,7 +156,7 @@ def worker(mongo, celery_session_worker, celery_inspect):
@pytest.fixture(scope='session')
-def mongo_infra(monkeysession):
+def mongo_infra():
return infrastructure.setup_mongo()
@@ -167,9 +168,9 @@ def mongo(mongo_infra):
@pytest.fixture(scope='session')
-def elastic_infra(monkeysession):
+def elastic_infra():
""" Provides elastic infrastructure to the session """
- monkeysession.setattr('nomad.config.elastic', config.elastic._replace(index_name='test_nomad_fairdi_calcs'))
+ config.elastic.index_name = 'test_nomad_fairdi_calcs'
try:
return infrastructure.setup_elastic()
except Exception:
@@ -201,7 +202,7 @@ def elastic(elastic_infra):
@contextmanager
-def create_postgres_infra(monkeysession=None, **kwargs):
+def create_postgres_infra(patch=None, **kwargs):
"""
A generator that sets up and tears down a test db and monkeypatches it to the
respective global infrastructure variables.
@@ -210,15 +211,11 @@ def create_postgres_infra(monkeysession=None, **kwargs):
db_args.update(**kwargs)
old_config = config.repository_db
- new_config = config.RepositoryDBConfig(
- old_config.host,
- old_config.port,
- db_args.get('dbname'),
- old_config.user,
- old_config.password)
+ new_config = config.NomadConfig(**config.repository_db)
+ new_config.update(**db_args)
- if monkeysession is not None:
- monkeysession.setattr('nomad.config.repository_db', new_config)
+ if patch is not None:
+ patch.setattr('nomad.config.repository_db', new_config)
connection, _ = infrastructure.sqlalchemy_repository_db(**db_args)
assert connection is not None
@@ -229,18 +226,18 @@ def create_postgres_infra(monkeysession=None, **kwargs):
db = Session(bind=connection, autocommit=True)
old_connection, old_db = None, None
- if monkeysession is not None:
+ if patch is not None:
from nomad.infrastructure import repository_db_conn, repository_db
old_connection, old_db = repository_db_conn, repository_db
- monkeysession.setattr('nomad.infrastructure.repository_db_conn', connection)
- monkeysession.setattr('nomad.infrastructure.repository_db', db)
+ patch.setattr('nomad.infrastructure.repository_db_conn', connection)
+ patch.setattr('nomad.infrastructure.repository_db', db)
yield db
- if monkeysession is not None:
- monkeysession.setattr('nomad.infrastructure.repository_db_conn', old_connection)
- monkeysession.setattr('nomad.infrastructure.repository_db', old_db)
- monkeysession.setattr('nomad.config.repository_db', old_config)
+ if patch is not None:
+ patch.setattr('nomad.infrastructure.repository_db_conn', old_connection)
+ patch.setattr('nomad.infrastructure.repository_db', old_db)
+ patch.setattr('nomad.config.repository_db', old_config)
trans.rollback()
db.expunge_all()
@@ -461,16 +458,8 @@ def smtpd(request):
@pytest.fixture(scope='function', autouse=True)
def mails(smtpd, monkeypatch):
smtpd.clear()
-
- old_config = config.mail
- new_config = config.MailConfig(
- 'localhost',
- old_config.port,
- old_config.user,
- old_config.password,
- old_config.from_address)
-
- monkeypatch.setattr('nomad.config.mail', new_config)
+ monkeypatch.setattr('nomad.config.mail.enabled', True)
+ monkeypatch.setattr('nomad.config.mail.host', 'localhost')
yield smtpd
diff --git a/tests/processing/test_data.py b/tests/processing/test_data.py
index badb9277a6be11270f40bb0062d4fb3890f52e7a..6341b1cc1606aab1797fd130cc1c12d244b5518d 100644
--- a/tests/processing/test_data.py
+++ b/tests/processing/test_data.py
@@ -25,7 +25,8 @@ from nomad.processing import Upload, Calc
from nomad.processing.base import task as task_decorator, FAILURE, SUCCESS
-def test_send_mail(mails):
+def test_send_mail(mails, monkeypatch):
+ monkeypatch.setattr('nomad.config.mail.enabled', True)
infrastructure.send_mail('test name', 'test@email.de', 'test message', 'subjct')
for message in mails.messages:
@@ -90,7 +91,7 @@ def assert_processing(upload: Upload):
assert upload_files.metadata.get(calc.calc_id) is not None
-def test_processing(processed, no_warn, mails):
+def test_processing(processed, no_warn, mails, monkeypatch):
assert_processing(processed)
assert len(mails.messages) == 1
diff --git a/tests/test_api.py b/tests/test_api.py
index 79589d9055fe052cb886305df8e05e703cf75077..2191667ccc407251f5d340ec6f456c809b22cc2c 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -22,7 +22,7 @@ import inspect
from passlib.hash import bcrypt
from datetime import datetime
-from nomad import config, coe_repo, search, parsing, files
+from nomad import coe_repo, search, parsing, files
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
@@ -48,12 +48,14 @@ def test_user_signature_token(client, test_user_auth):
class TestAdmin:
@pytest.mark.timeout(10)
- def test_reset(self, client, admin_user_auth, expandable_postgres):
+ def test_reset(self, client, admin_user_auth, expandable_postgres, monkeypatch):
+ monkeypatch.setattr('nomad.config.services.disable_reset', False)
rv = client.post('/admin/reset', headers=admin_user_auth)
assert rv.status_code == 200
@pytest.mark.timeout(10)
- def test_remove(self, client, admin_user_auth, expandable_postgres):
+ def test_remove(self, client, admin_user_auth, expandable_postgres, monkeypatch):
+ monkeypatch.setattr('nomad.config.services.disable_reset', False)
rv = client.post('/admin/remove', headers=admin_user_auth)
assert rv.status_code == 200
@@ -65,22 +67,7 @@ class TestAdmin:
rv = client.post('/admin/reset', headers=test_user_auth)
assert rv.status_code == 401
- @pytest.fixture(scope='function')
- def disable_reset(self, monkeypatch):
- old_config = config.services
- new_config = config.NomadServicesConfig(
- config.services.api_host,
- config.services.api_port,
- config.services.api_base_path,
- config.services.api_secret,
- config.services.admin_password,
- config.services.upload_url,
- True)
- monkeypatch.setattr(config, 'services', new_config)
- yield None
- monkeypatch.setattr(config, 'services', old_config)
-
- def test_disabled(self, client, admin_user_auth, disable_reset, postgres):
+ def test_disabled(self, client, admin_user_auth, postgres):
rv = client.post('/admin/reset', headers=admin_user_auth)
assert rv.status_code == 400