config.py 5.94 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
17
18
19
"""
This module is used to store all configuration values. It makes use of
*namedtuples* to create key sensitive configuration objects.
"""

20
import os
Markus Scheidgen's avatar
Markus Scheidgen committed
21
import logging
22
from collections import namedtuple
Markus Scheidgen's avatar
Markus Scheidgen committed
23
24
25
26
27
import warnings

warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

28

29
FilesConfig = namedtuple(
30
    'FilesConfig', ['uploads_bucket', 'raw_bucket', 'archive_bucket', 'staging_bucket', 'public_bucket'])
31
32
""" API independent configuration for the object storage. """

33
CeleryConfig = namedtuple('Celery', ['broker_url', 'max_memory', 'timeout'])
Markus Scheidgen's avatar
Markus Scheidgen committed
34
""" Used to configure the RabbitMQ for celery. """
35

36
FSConfig = namedtuple('FSConfig', ['tmp', 'objects', 'nomad_tmp'])
37
38
""" Used to configure file stystem access. """

39
40
41
RepositoryDBConfig = namedtuple('RepositoryDBConfig', ['host', 'port', 'dbname', 'user', 'password'])
""" Used to configure access to NOMAD-coe repository db. """

Markus Scheidgen's avatar
Markus Scheidgen committed
42
ElasticConfig = namedtuple('ElasticConfig', ['host', 'port', 'index_name'])
43
44
""" Used to configure elastic search. """

45
MongoConfig = namedtuple('MongoConfig', ['host', 'port', 'db_name'])
46
47
""" Used to configure mongo db. """

Markus Scheidgen's avatar
Markus Scheidgen committed
48
LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port', 'level'])
49
""" Used to configure and enable/disable the ELK based centralized logging. """
50

51
NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_host', 'api_port', 'api_base_path', 'api_secret', 'admin_password', 'upload_url', 'disable_reset'])
Markus Scheidgen's avatar
Markus Scheidgen committed
52
53
""" Used to configure nomad services: worker, handler, api """

54
MailConfig = namedtuple('MailConfig', ['host', 'port', 'user', 'password', 'from_address'])
55
56
""" Used to configure how nomad can send email """

57
files = FilesConfig(
58
    uploads_bucket='uploads',
59
    raw_bucket=os.environ.get('NOMAD_FILES_RAW_BUCKET', 'raw'),
60
    archive_bucket='archive',
61
62
    staging_bucket='staging',
    public_bucket='public'
63
)
64
65
66
67
68
69
70

rabbit_host = os.environ.get('NOMAD_RABBITMQ_HOST', 'localhost')
rabbit_port = os.environ.get('NOMAD_RABBITMQ_PORT', None)
rabbit_user = 'rabbitmq'
rabbit_password = 'rabbitmq'
rabbit_url = 'pyamqp://%s:%s@%s//' % (rabbit_user, rabbit_password, rabbit_host)

71
72
73
74
75
76
77
78
79
80
81
82

def get_loglevel_from_env(key, default_level=logging.INFO):
    plain_value = os.environ.get(key, None)
    if plain_value is None:
        return default_level
    else:
        try:
            return int(plain_value)
        except ValueError:
            return getattr(logging, plain_value, default_level)


83
celery = CeleryConfig(
84
85
86
    broker_url=rabbit_url,
    max_memory=int(os.environ.get('NOMAD_CELERY_MAXMEMORY', 64e6)),  # 64 GB
    timeout=int(os.environ.get('NOMAD_CELERY_TIMEOUT', 3 * 3600))  # 3h
87
)
88

89
fs = FSConfig(
90
    tmp=os.environ.get('NOMAD_FILES_TMP_DIR', '.volumes/fs/tmp'),
91
92
    objects=os.environ.get('NOMAD_FILES_OBJECTS_DIR', '.volumes/fs/objects'),
    nomad_tmp=os.environ.get('NOMAD_FILES_NOMAD_TMP_DIR', '/nomad/tmp')
93
)
94
elastic = ElasticConfig(
95
    host=os.environ.get('NOMAD_ELASTIC_HOST', 'localhost'),
96
    port=int(os.environ.get('NOMAD_ELASTIC_PORT', 9200)),
Markus Scheidgen's avatar
Markus Scheidgen committed
97
    index_name=os.environ.get('NOMAD_ELASTIC_INDEX_NAME', 'nomad_fairdi_calcs')
98
)
99
repository_db = RepositoryDBConfig(
100
101
    host=os.environ.get('NOMAD_COE_REPO_DB_HOST', 'localhost'),
    port=int(os.environ.get('NOMAD_COE_REPO_DB_PORT', 5432)),
Markus Scheidgen's avatar
Markus Scheidgen committed
102
    dbname=os.environ.get('NOMAD_COE_REPO_DB_NAME', 'nomad_fairdi_repo_db'),
103
104
    user=os.environ.get('NOMAD_COE_REPO_DB_USER', 'postgres'),
    password=os.environ.get('NOMAD_COE_REPO_PASSWORD', 'nomad')
105
)
106
mongo = MongoConfig(
107
    host=os.environ.get('NOMAD_MONGO_HOST', 'localhost'),
Markus Scheidgen's avatar
Markus Scheidgen committed
108
    port=int(os.environ.get('NOMAD_MONGO_PORT', 27017)),
Markus Scheidgen's avatar
Markus Scheidgen committed
109
    db_name=os.environ.get('NOMAD_MONGO_DB_NAME', 'nomad_fairdi')
110
)
111
logstash = LogstashConfig(
112
    enabled=True,
113
    host=os.environ.get('NOMAD_LOGSTASH_HOST', 'localhost'),
Markus Scheidgen's avatar
Markus Scheidgen committed
114
    tcp_port=int(os.environ.get('NOMAD_LOGSTASH_TCPPORT', '5000')),
115
    level=get_loglevel_from_env('NOMAD_LOGSTASH_LEVEL', default_level=logging.DEBUG)
116
)
Markus Scheidgen's avatar
Markus Scheidgen committed
117
services = NomadServicesConfig(
118
119
    api_host=os.environ.get('NOMAD_API_HOST', 'localhost'),
    api_port=int(os.environ.get('NOMAD_API_PORT', 8000)),
120
    api_base_path=os.environ.get('NOMAD_API_BASE_PATH', '/nomad/api'),
121
122
    api_secret=os.environ.get('NOMAD_API_SECRET', 'defaultApiSecret'),
    admin_password=os.environ.get('NOMAD_API_ADMIN_PASSWORD', 'password'),
123
    upload_url=os.environ.get('NOMAD_UPLOAD_URL', 'http://localhost/nomad/uploads'),
124
    disable_reset=os.environ.get('NOMAD_API_DISABLE_RESET', 'false') == 'true'
Markus Scheidgen's avatar
Markus Scheidgen committed
125
)
126
127
128
129
130
131
132
migration_source_db = RepositoryDBConfig(
    host=os.environ.get('NOMAD_MIGRATION_SOURCE_DB_HOST', 'db-repository.nomad.esc'),
    port=int(os.environ.get('NOMAD_MIGRATION_SOURCE_DB_PORT', 5432)),
    dbname=os.environ.get('NOMAD_MIGRATION_SOURCE_DB_NAME', 'nomad_prod'),
    user=os.environ.get('NOMAD_MIGRATION_SOURCE_USER', 'nomadlab'),
    password=os.environ.get('NOMAD_MIGRATION_SOURCE_PASSWORD', '*')
)
133
mail = MailConfig(
134
    host=os.environ.get('NOMAD_SMTP_HOST', ''),  # empty or None host disables email
135
136
137
138
139
    port=int(os.environ.get('NOMAD_SMTP_PORT', 8995)),
    user=os.environ.get('NOMAD_SMTP_USER', None),
    password=os.environ.get('NOMAD_SMTP_PASSWORD', None),
    from_address=os.environ.get('NOMAD_MAIL_FROM', 'webmaster@nomad-coe.eu')
)
Markus Scheidgen's avatar
Markus Scheidgen committed
140

Markus Scheidgen's avatar
Markus Scheidgen committed
141
console_log_level = get_loglevel_from_env('NOMAD_CONSOLE_LOGLEVEL', default_level=logging.WARNING)
142
service = os.environ.get('NOMAD_SERVICE', 'unknown nomad service')
143
release = os.environ.get('NOMAD_RELEASE', 'devel')