Skip to content

GitLab

Commit ec062073 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored configuration to work on yml and environment.

parent d22785ab
Hide whitespace changes
Inline Side-by-side
.vscode/.gitignore
View file @ ec062073
.ropeproject/
*.sql
\ No newline at end of file
*.sql
launch.json
\ No newline at end of file
.vscode/launch.json deleted 100644 → 0
View file @ d22785ab
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "chrome",
"request": "launch",
"name": "Launch Chrome against localhost",
"url": "http://localhost:3000",
"webRoot": "${workspaceFolder}/gui"
},
{
"name": "Python: API Flask (0.11.x or later)",
"type": "python",
"request": "launch",
"module": "flask",
"env": {
"FLASK_APP": "nomad/api.py"
},
"args": [
"run",
"--port", "8000",
"--no-debugger",
"--no-reload"
]
},
{
"name": "Python: worker nomad.processing",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/celery",
"args": [
"worker", "-l" , "debug", "-A", "nomad.processing"
]
},
{
"name": "Python: some test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_api.py::TestRepo::test_search[2-user-other_test_user]"
]
},
{
"name": "Python: crystal normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/crystal-tests/data/parsers/crystal/si.out]"
]
},
{
"name": "Python: cp2k normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/cp2k-tests/data/parsers/cp2k/si_bulk8.out]"
]
},
{
"name": "Python: cpmd normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/cpmd-tests/data/parsers/cpmd/geo_output.out]"
]
},
{
"name": "Python: wien2k normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/wien2k-tests/data/parsers/wien2k/AlN/AlN_ZB.scf]"
]
},
{
"name": "Python: gaussian normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/gaussian-tests/data/parsers/gaussian/Al.out]"
]
},
{
"name": "Python: aniline gaussian normalizer test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/gaussian-tests/data/parsers/gaussian/al-1.out]"
]
},
{
"name": "Python: test_parsing match test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_parsing.py::test_match"
]
},
{
"name": "Python: nwchem normalizer test h2o sp test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/nwchem-tests/data/parsers/nwchem/sp_output.out]"
]
},
{
"name": "Python: Vasp XML test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/vasp-tests/data/parsers/vasp/vasp.xml]"
]
},
{
"name": "Quantum Espresso Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/quantumespresso-tests/data/parsers/quantum-espresso/W.out]"
]
},
{
"name": "Abinit Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/abinit-tests/data/parsers/abinit/Fe.out]"
]
},
{
"name": "Castep Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/castep-tests/data/parsers/castep/BC2N-Pmm2-Raman.castep]"
]
},
{
"name": "DL-Poly Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/dl-poly-tests/data/parsers/dl-poly/OUTPUT]"
]
},
{
"name": "Lib Atoms Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/lib-atoms-tests/data/parsers/lib-atoms/gp.xml]"
]
},
{
"name": "Octopus Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/octopus-tests/data/parsers/octopus/stdout.txt]"
]
},
{
"name": "Python: Current File",
"type": "python",
"request": "launch",
"program": "${file}"
},
{
"name": "Python: Attach",
"type": "python",
"request": "attach",
"localRoot": "${workspaceFolder}",
"remoteRoot": "${workspaceFolder}",
"port": 3000,
"secret": "my_secret",
"host": "localhost"
}
]
}
\ No newline at end of file
nomad/client/local.py
View file @ ec062073
......@@ -26,7 +26,7 @@ from nomad.files import ArchiveBasedStagingUploadFiles
from nomad.parsing import parser_dict, LocalBackend, match_parser
from nomad.normalizing import normalizers
from .main import cli, get_nomad_url
from .main import cli
class CalcProcReproduction:
......@@ -67,7 +67,7 @@ class CalcProcReproduction:
# TODO currently only downloads mainfile
self.logger.info('Downloading calc.', mainfile=self.mainfile)
token = client.auth.get_user().response().result.token
req = requests.get('%s/raw/%s/%s' % (get_nomad_url(), self.upload_id, os.path.dirname(self.mainfile)) + '/*', stream=True, headers={'X-Token': token})
req = requests.get('%s/raw/%s/%s' % (config.client.url, self.upload_id, os.path.dirname(self.mainfile)) + '/*', stream=True, headers={'X-Token': token})
with open(local_path, 'wb') as f:
for chunk in req.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE):
f.write(chunk)
......@@ -166,7 +166,7 @@ class CalcProcReproduction:
def local(archive_id, show_backend=False, show_metadata=False, **kwargs):
print(kwargs)
utils.configure_logging()
utils.get_logger(__name__).info('Using %s' % get_nomad_url())
utils.get_logger(__name__).info('Using %s' % config.client.url)
with CalcProcReproduction(archive_id, **kwargs) as local:
backend = local.parse()
local.normalize_all(parser_backend=backend)
......
nomad/client/main.py
View file @ ec062073
......@@ -24,16 +24,6 @@ from urllib.parse import urlparse
from nomad import config, utils, infrastructure
_default_url = 'http://%s:%d/%s' % (config.services.api_host, config.services.api_port, config.services.api_base_path.strip('/'))
_nomad_url = os.environ.get('NOMAD_URL', _default_url)
_user = os.environ.get('NOMAD_USER', 'leonard.hofstadter@nomad-fairdi.tests.de')
_pw = os.environ.get('NOMAD_PASSWORD', 'password')
def get_nomad_url():
return _nomad_url
def create_client():
return _create_client()
......@@ -42,15 +32,15 @@ def _create_client(*args, **kwargs):
return __create_client(*args, **kwargs)
def __create_client(user: str = _user, password: str = _pw):
def __create_client(user: str = config.client.user, password: str = config.client.password):
""" A factory method to create the client. """
host = urlparse(_nomad_url).netloc.split(':')[0]
host = urlparse(config.client.url).netloc.split(':')[0]
http_client = RequestsClient()
if user is not None:
http_client.set_basic_auth(host, user, password)
client = SwaggerClient.from_url(
'%s/swagger.json' % _nomad_url,
'%s/swagger.json' % config.client.url,
http_client=http_client)
utils.get_logger(__name__).info('created bravado client', user=user)
......@@ -65,15 +55,15 @@ def handle_common_errors(func):
except requests.exceptions.ConnectionError:
click.echo(
'\nCould not connect to nomad at %s. '
'Check connection and url.' % _nomad_url)
'Check connection and url.' % config.client.url)
sys.exit(0)
return wrapper
@click.group()
@click.option('-n', '--url', default=_nomad_url, help='The URL where nomad is running "%s".' % _nomad_url)
@click.option('-u', '--user', default=None, help='the user name to login, default no login.')
@click.option('-w', '--password', default=_pw, help='the password use to login.')
@click.option('-n', '--url', default=config.client.url, help='The URL where nomad is running, default is "%s".' % config.client.url)
@click.option('-u', '--user', default=None, help='the user name to login, default is "%s" login.' % config.client.user)
@click.option('-w', '--password', default=config.client.password, help='the password used to login.')
@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
@click.option('--debug', help='sets log level to debug', is_flag=True)
def cli(url: str, verbose: bool, debug: bool, user: str, password: str):
......@@ -92,8 +82,7 @@ def cli(url: str, verbose: bool, debug: bool, user: str, password: str):
logger.info('Used nomad is %s' % url)
logger.info('Used user is %s' % user)
global _nomad_url
_nomad_url = url
config.client.url = url
global _create_client
......
nomad/client/upload.py
View file @ ec062073
......@@ -19,17 +19,16 @@ import click
import urllib.parse
import requests
from nomad import utils
from nomad import utils, config
from nomad.processing import FAILURE, SUCCESS
from .main import cli, create_client
from nomad.client import main
def stream_upload_with_client(client, stream, name=None):
user = client.auth.get_user().response().result
token = user.token
url = main._nomad_url + '/uploads/'
url = config.client.url + '/uploads/'
if name is not None:
url += '?name=%s' % urllib.parse.quote(name)
......
nomad/config.py
View file @ ec062073
......@@ -12,68 +12,132 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
This module is used to store all configuration values. It makes use of
*namedtuples* to create key sensitive configuration objects.
"""
import os
import logging
from collections import namedtuple
import os
import os.path
import yaml
import warnings
from nomad import utils
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
class NomadConfig(utils.POPO):
pass
# class ConfigProperty:
# def __init__(
# self, name: str, default_value: Union[int, str, bool], help: str = None,
# env_var: str = None) -> None:
# self.name = name
# self.default_value = default_value,
# self.help = help
# self.env_var = env_var
CELERY_WORKER_ROUTING = 'worker'
CELERY_QUEUE_ROUTING = 'queue'
CeleryConfig = namedtuple('Celery', [
'broker_url', 'max_memory', 'timeout', 'acks_late', 'routing'])
"""
Used to configure the RabbitMQ for celery.
rabbitmq = NomadConfig(
host='localhost',
user='rabbitmq',
password='rabbitmq'
)
Arguments:
broker_url: The rabbitmq broker URL
max_memory: Max worker memory
timeout: Task timeout
acks_late: Use celery acks_late if set
routing: Set to ``queue`` for routing via upload, calc queues. Processing tasks for one
upload might be routed to different worker and all staging files must be accessible
for all workers via distributed fs. Set to ``worker`` to route all tasks related
to the same upload to the same worker.
"""
FSConfig = namedtuple('FSConfig', ['tmp', 'staging', 'public', 'prefix_size'])
""" Used to configure file stystem access. """
def rabbitmq_url():
return 'pyamqp://%s:%s@%s//' % (rabbitmq.user, rabbitmq.password, rabbitmq.host)
RepositoryDBConfig = namedtuple('RepositoryDBConfig', ['host', 'port', 'dbname', 'user', 'password'])
""" Used to configure access to NOMAD-coe repository db. """
ElasticConfig = namedtuple('ElasticConfig', ['host', 'port', 'index_name'])
""" Used to configure elastic search. """
celery = NomadConfig(
max_memory=64e6, # 64 GB
timeout=1800, # 1/2 h
acks_late=True,
routing=CELERY_QUEUE_ROUTING
)
MongoConfig = namedtuple('MongoConfig', ['host', 'port', 'db_name'])
""" Used to configure mongo db. """
fs = NomadConfig(
tmp='.volumes/fs/tmp',
staging='.volumes/fs/staging',
public='.volumes/fs/public',
prefix_size=2
)
LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port', 'level'])
""" Used to configure and enable/disable the ELK based centralized logging. """
elastic = NomadConfig(
host='localhost',
port=9200,
index_name='nomad_fairdi_calcs'
)
NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_host', 'api_port', 'api_base_path', 'api_secret', 'admin_password', 'upload_url', 'disable_reset'])
""" Used to configure nomad services: worker, handler, api """
repository_db = NomadConfig(
host='localhost',
port=5432,
dbname='nomad_fairdi_repo_db',
user='postgres',
password='nomad'
)
MailConfig = namedtuple('MailConfig', ['host', 'port', 'user', 'password', 'from_address'])
""" Used to configure how nomad can send email """
mongo = NomadConfig(
host='localhost',
port=27017,
db_name='nomad_fairdi'
)
NormalizeConfig = namedtuple('NormalizeConfig', ['all_systems'])
""" Used to configure the normalizers """
logstash = NomadConfig(
enabled=True,
host='localhost',
tcp_port='5000',
level=logging.DEBUG
)
rabbit_host = os.environ.get('NOMAD_RABBITMQ_HOST', 'localhost')
rabbit_port = os.environ.get('NOMAD_RABBITMQ_PORT', None)
rabbit_user = 'rabbitmq'
rabbit_password = 'rabbitmq'
rabbit_url = 'pyamqp://%s:%s@%s//' % (rabbit_user, rabbit_password, rabbit_host)
services = NomadConfig(
api_host='localhost',
api_port=8000,
api_base_path='/nomad/api',
api_secret='defaultApiSecret',
admin_password='password',
disable_reset=True
)
def upload_url():
return 'http://%s:%s/%s/uploads' % (services.api_host, services.api_port, services.api_base_path[:-3])
migration_source_db = NomadConfig(
host='db-repository.nomad.esc',
port=5432,
dbname='nomad_prod',
user='nomadlab',
password='*'
)
mail = NomadConfig(
enabled=False,
with_login=False,
host='',
port=8995,
user='',
password='',
from_address='webmaster@nomad-coe.eu'
)
normalize = NomadConfig(
all_systems=False
)
client = NomadConfig(
user='leonard.hofstadter@nomad-fairdi.tests.de',
password='password',
url='http://localhost:8000/nomad/api'
)
console_log_level = logging.WARNING
service = 'unknown nomad service'
release = 'devel'
auxfile_cutoff = 30
def get_loglevel_from_env(key, default_level=logging.INFO):
......@@ -84,78 +148,113 @@ def get_loglevel_from_env(key, default_level=logging.INFO):
try:
return int(plain_value)
except ValueError:
return getattr(logging, plain_value, default_level)
return getattr(logging, plain_value)
celery = CeleryConfig(
broker_url=rabbit_url,
max_memory=int(os.environ.get('NOMAD_CELERY_MAXMEMORY', 64e6)), # 64 GB
timeout=int(os.environ.get('NOMAD_CELERY_TIMEOUT', 1800)), # 1/2h
acks_late=bool(os.environ.get('NOMAD_CELERY_ACKS_LATE', True)),
routing=os.environ.get('NOMAD_CELERY_ROUTING', CELERY_QUEUE_ROUTING)
)
fs = FSConfig(
tmp=os.environ.get('NOMAD_FILES_TMP_DIR', '.volumes/fs/tmp'),
staging=os.environ.get('NOMAD_FILES_STAGING_DIR', '.volumes/fs/staging'),
public=os.environ.get('NOMAD_FILES_PUBLIC_DIR', '.volumes/fs/public'),
prefix_size=int(os.environ.get('NOMAD_FILES_PREFIX_SIZE', 2))
)
elastic = ElasticConfig(
host=os.environ.get('NOMAD_ELASTIC_HOST', 'localhost'),
port=int(os.environ.get('NOMAD_ELASTIC_PORT', 9200)),
index_name=os.environ.get('NOMAD_ELASTIC_INDEX_NAME', 'nomad_fairdi_calcs')
)
repository_db = RepositoryDBConfig(
host=os.environ.get('NOMAD_COE_REPO_DB_HOST', 'localhost'),
port=int(os.environ.get('NOMAD_COE_REPO_DB_PORT', 5432)),
dbname=os.environ.get('NOMAD_COE_REPO_DB_NAME', 'nomad_fairdi_repo_db'),
user=os.environ.get('NOMAD_COE_REPO_DB_USER', 'postgres'),
password=os.environ.get('NOMAD_COE_REPO_PASSWORD', 'nomad')
)
mongo = MongoConfig(
host=os.environ.get('NOMAD_MONGO_HOST', 'localhost'),
port=int(os.environ.get('NOMAD_MONGO_PORT', 27017)),
db_name=os.environ.get('NOMAD_MONGO_DB_NAME', 'nomad_fairdi')
)
logstash = LogstashConfig(
enabled=True,
host=os.environ.get('NOMAD_LOGSTASH_HOST', 'localhost'),
tcp_port=int(os.environ.get('NOMAD_LOGSTASH_TCPPORT', '5000')),
level=get_loglevel_from_env('NOMAD_LOGSTASH_LEVEL', default_level=logging.DEBUG)
)
services = NomadServicesConfig(
api_host=os.environ.get('NOMAD_API_HOST', 'localhost'),
api_port=int(os.environ.get('NOMAD_API_PORT', 8000)),
api_base_path=os.environ.get('NOMAD_API_BASE_PATH', '/nomad/api'),
api_secret=os.environ.get('NOMAD_API_SECRET', 'defaultApiSecret'),
admin_password=os.environ.get('NOMAD_API_ADMIN_PASSWORD', 'password'),
upload_url=os.environ.get('NOMAD_UPLOAD_URL', 'http://localhost/nomad/uploads'),
disable_reset=os.environ.get('NOMAD_API_DISABLE_RESET', 'false') == 'true'
)
migration_source_db = RepositoryDBConfig(
host=os.environ.get('NOMAD_MIGRATION_SOURCE_DB_HOST', 'db-repository.nomad.esc'),
port=int(os.environ.get('NOMAD_MIGRATION_SOURCE_DB_PORT', 5432)),
dbname=os.environ.get('NOMAD_MIGRATION_SOURCE_DB_NAME', 'nomad_prod'),
user=os.environ.get('NOMAD_MIGRATION_SOURCE_USER', 'nomadlab'),
password=os.environ.get('NOMAD_MIGRATION_SOURCE_PASSWORD', '*')
)
mail = MailConfig(
host=os.environ.get('NOMAD_SMTP_HOST', ''), # empty or None host disables email
port=int(os.environ.get('NOMAD_SMTP_PORT', 8995)),
user=os.environ.get('NOMAD_SMTP_USER', None),
password=os.environ.get('NOMAD_SMTP_PASSWORD', None),
from_address=os.environ.get('NOMAD_MAIL_FROM', 'webmaster@nomad-coe.eu')
)
normalize = NormalizeConfig(
all_systems=False
)
transformations = {
'console_log_level': get_loglevel_from_env,
'logstash_level': get_loglevel_from_env
}
console_log_level = get_loglevel_from_env('NOMAD_CONSOLE_LOGLEVEL', default_level=logging.WARNING)
service = os.environ.get('NOMAD_SERVICE', 'unknown nomad service')
release = os.environ.get('NOMAD_RELEASE', 'devel')
auxfile_cutoff = 30
"""
Number of max auxfiles. More auxfiles means no auxfiles, but probably a directory of many
mainfiles.
"""
# use std python logger, since logging is not configured while loading configuration
logger = logging.getLogger(__name__)
def apply(config, key, value) -> None:
"""
Changes the config according to given key and value. The keys are interpreted as paths
to config values with ``_`` as a separator. E.g. ``fs_staging`` leading to
``config.fs.staging``
"""
path = list(reversed(key.split('_')))
child_segment