config.py 5.15 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
17
18
19
"""
This module is used to store all configuration values. It makes use of
*namedtuples* to create key sensitive configuration objects.
"""

20
import os
Markus Scheidgen's avatar
Markus Scheidgen committed
21
import logging
22
from collections import namedtuple
Markus Scheidgen's avatar
Markus Scheidgen committed
23
24
25
26
27
import warnings

warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

28

29
FilesConfig = namedtuple(
30
    'FilesConfig', ['uploads_bucket', 'raw_bucket', 'archive_bucket', 'staging_bucket', 'public_bucket'])
31
32
""" API independent configuration for the object storage. """

Markus Scheidgen's avatar
Markus Scheidgen committed
33
CeleryConfig = namedtuple('Celery', ['broker_url'])
Markus Scheidgen's avatar
Markus Scheidgen committed
34
""" Used to configure the RabbitMQ for celery. """
35

Markus Scheidgen's avatar
Markus Scheidgen committed
36
FSConfig = namedtuple('FSConfig', ['tmp', 'objects'])
37
38
""" Used to configure file stystem access. """

39
40
41
RepositoryDBConfig = namedtuple('RepositoryDBConfig', ['host', 'port', 'dbname', 'user', 'password'])
""" Used to configure access to NOMAD-coe repository db. """

42
ElasticConfig = namedtuple('ElasticConfig', ['host', 'port', 'index_name', 'coe_repo_calcs_index_name'])
43
44
""" Used to configure elastic search. """

45
MongoConfig = namedtuple('MongoConfig', ['host', 'port', 'db_name'])
46
47
""" Used to configure mongo db. """

Markus Scheidgen's avatar
Markus Scheidgen committed
48
LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port', 'level'])
49
""" Used to configure and enable/disable the ELK based centralized logging. """
50

51
NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_host', 'api_port', 'api_base_path', 'api_secret', 'admin_password', 'disable_reset'])
Markus Scheidgen's avatar
Markus Scheidgen committed
52
53
""" Used to configure nomad services: worker, handler, api """

54
files = FilesConfig(
55
    uploads_bucket='uploads',
56
    raw_bucket=os.environ.get('NOMAD_FILES_RAW_BUCKET', 'raw'),
57
    archive_bucket='archive',
58
59
    staging_bucket='staging',
    public_bucket='public'
60
)
61
62
63
64
65
66
67

rabbit_host = os.environ.get('NOMAD_RABBITMQ_HOST', 'localhost')
rabbit_port = os.environ.get('NOMAD_RABBITMQ_PORT', None)
rabbit_user = 'rabbitmq'
rabbit_password = 'rabbitmq'
rabbit_url = 'pyamqp://%s:%s@%s//' % (rabbit_user, rabbit_password, rabbit_host)

68
69
70
71
72
73
74
75
76
77
78
79

def get_loglevel_from_env(key, default_level=logging.INFO):
    plain_value = os.environ.get(key, None)
    if plain_value is None:
        return default_level
    else:
        try:
            return int(plain_value)
        except ValueError:
            return getattr(logging, plain_value, default_level)


80
celery = CeleryConfig(
Markus Scheidgen's avatar
Markus Scheidgen committed
81
    broker_url=rabbit_url
82
)
83

84
fs = FSConfig(
85
86
    tmp=os.environ.get('NOMAD_FILES_TMP_DIR', '.volumes/fs/tmp'),
    objects=os.environ.get('NOMAD_FILES_OBJECTS_DIR', '.volumes/fs/objects')
87
)
88
elastic = ElasticConfig(
89
    host=os.environ.get('NOMAD_ELASTIC_HOST', 'localhost'),
90
    port=int(os.environ.get('NOMAD_ELASTIC_PORT', 9200)),
91
92
    index_name=os.environ.get('NOMAD_ELASTIC_INDEX_NAME', 'calcs'),
    coe_repo_calcs_index_name='coe_repo_calcs'
93
)
94
repository_db = RepositoryDBConfig(
95
96
97
98
99
    host=os.environ.get('NOMAD_COE_REPO_DB_HOST', 'localhost'),
    port=int(os.environ.get('NOMAD_COE_REPO_DB_PORT', 5432)),
    dbname=os.environ.get('NOMAD_COE_REPO_DB_NAME', 'nomad'),
    user=os.environ.get('NOMAD_COE_REPO_DB_USER', 'postgres'),
    password=os.environ.get('NOMAD_COE_REPO_PASSWORD', 'nomad')
100
)
101
mongo = MongoConfig(
102
    host=os.environ.get('NOMAD_MONGO_HOST', 'localhost'),
Markus Scheidgen's avatar
Markus Scheidgen committed
103
    port=int(os.environ.get('NOMAD_MONGO_PORT', 27017)),
104
    db_name=os.environ.get('NOMAD_MONGO_DB_NAME', 'users')
105
)
106
logstash = LogstashConfig(
107
    enabled=True,
108
    host=os.environ.get('NOMAD_LOGSTASH_HOST', 'localhost'),
Markus Scheidgen's avatar
Markus Scheidgen committed
109
    tcp_port=int(os.environ.get('NOMAD_LOGSTASH_TCPPORT', '5000')),
110
    level=get_loglevel_from_env('NOMAD_LOGSTASH_LEVEL', default_level=logging.DEBUG)
111
)
Markus Scheidgen's avatar
Markus Scheidgen committed
112
services = NomadServicesConfig(
113
114
    api_host=os.environ.get('NOMAD_API_HOST', 'localhost'),
    api_port=int(os.environ.get('NOMAD_API_PORT', 8000)),
115
    api_base_path=os.environ.get('NOMAD_API_BASE_PATH', '/nomad/api'),
116
117
    api_secret=os.environ.get('NOMAD_API_SECRET', 'defaultApiSecret'),
    admin_password=os.environ.get('NOMAD_API_ADMIN_PASSWORD', 'password'),
118
    disable_reset=os.environ.get('NOMAD_API_DISABLE_RESET', 'false') == 'true'
Markus Scheidgen's avatar
Markus Scheidgen committed
119
)
120
121
122
123
124
125
126
migration_source_db = RepositoryDBConfig(
    host=os.environ.get('NOMAD_MIGRATION_SOURCE_DB_HOST', 'db-repository.nomad.esc'),
    port=int(os.environ.get('NOMAD_MIGRATION_SOURCE_DB_PORT', 5432)),
    dbname=os.environ.get('NOMAD_MIGRATION_SOURCE_DB_NAME', 'nomad_prod'),
    user=os.environ.get('NOMAD_MIGRATION_SOURCE_USER', 'nomadlab'),
    password=os.environ.get('NOMAD_MIGRATION_SOURCE_PASSWORD', '*')
)
Markus Scheidgen's avatar
Markus Scheidgen committed
127

128
console_log_level = get_loglevel_from_env('NOMAD_CONSOLE_LOGLEVEL', default_level=logging.INFO)
129
service = os.environ.get('NOMAD_SERVICE', 'unknown nomad service')
130
release = os.environ.get('NOMAD_RELEASE', 'devel')