config.py 6.61 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
17
18
19
"""
This module is used to store all configuration values. It makes use of
*namedtuples* to create key sensitive configuration objects.
"""

20
import os
Markus Scheidgen's avatar
Markus Scheidgen committed
21
import logging
22
from collections import namedtuple
Markus Scheidgen's avatar
Markus Scheidgen committed
23
24
25
26
27
import warnings

warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

28

29
30
CELERY_WORKER_ROUTING = 'worker'
CELERY_QUEUE_ROUTING = 'queue'
31

32
33
CeleryConfig = namedtuple('Celery', [
    'broker_url', 'max_memory', 'timeout', 'acks_late', 'routing'])
34
35
36
37
38
39
40
41
42
43
44
45
46
"""
Used to configure the RabbitMQ for celery.

Arguments:
    broker_url: The rabbitmq broker URL
    max_memory: Max worker memory
    timeout: Task timeout
    acks_late: Use celery acks_late if set
    routing: Set to ``queue`` for routing via upload, calc queues. Processing tasks for one
        upload might be routed to different worker and all staging files must be accessible
        for all workers via distributed fs. Set to ``worker`` to route all tasks related
        to the same upload to the same worker.
"""
47

48
FSConfig = namedtuple('FSConfig', ['tmp', 'staging', 'public', 'prefix_size'])
49
50
""" Used to configure file stystem access. """

51
52
53
RepositoryDBConfig = namedtuple('RepositoryDBConfig', ['host', 'port', 'dbname', 'user', 'password'])
""" Used to configure access to NOMAD-coe repository db. """

Markus Scheidgen's avatar
Markus Scheidgen committed
54
ElasticConfig = namedtuple('ElasticConfig', ['host', 'port', 'index_name'])
55
56
""" Used to configure elastic search. """

57
MongoConfig = namedtuple('MongoConfig', ['host', 'port', 'db_name'])
58
59
""" Used to configure mongo db. """

Markus Scheidgen's avatar
Markus Scheidgen committed
60
LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port', 'level'])
61
""" Used to configure and enable/disable the ELK based centralized logging. """
62

63
NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_host', 'api_port', 'api_base_path', 'api_secret', 'admin_password', 'upload_url', 'disable_reset'])
Markus Scheidgen's avatar
Markus Scheidgen committed
64
65
""" Used to configure nomad services: worker, handler, api """

66
MailConfig = namedtuple('MailConfig', ['host', 'port', 'user', 'password', 'from_address'])
67
68
""" Used to configure how nomad can send email """

Markus Scheidgen's avatar
Markus Scheidgen committed
69
70
71
NormalizeConfig = namedtuple('NormalizeConfig', ['all_systems'])
""" Used to configure the normalizers """

72
73
74
75
76
77
rabbit_host = os.environ.get('NOMAD_RABBITMQ_HOST', 'localhost')
rabbit_port = os.environ.get('NOMAD_RABBITMQ_PORT', None)
rabbit_user = 'rabbitmq'
rabbit_password = 'rabbitmq'
rabbit_url = 'pyamqp://%s:%s@%s//' % (rabbit_user, rabbit_password, rabbit_host)

78
79
80
81
82
83
84
85
86
87
88
89

def get_loglevel_from_env(key, default_level=logging.INFO):
    plain_value = os.environ.get(key, None)
    if plain_value is None:
        return default_level
    else:
        try:
            return int(plain_value)
        except ValueError:
            return getattr(logging, plain_value, default_level)


90
celery = CeleryConfig(
91
92
    broker_url=rabbit_url,
    max_memory=int(os.environ.get('NOMAD_CELERY_MAXMEMORY', 64e6)),  # 64 GB
93
    timeout=int(os.environ.get('NOMAD_CELERY_TIMEOUT', 1800)),  # 1/2h
94
    acks_late=bool(os.environ.get('NOMAD_CELERY_ACKS_LATE', True)),
95
    routing=os.environ.get('NOMAD_CELERY_ROUTING', CELERY_QUEUE_ROUTING)
96
97
)
fs = FSConfig(
98
    tmp=os.environ.get('NOMAD_FILES_TMP_DIR', '.volumes/fs/tmp'),
99
    staging=os.environ.get('NOMAD_FILES_STAGING_DIR', '.volumes/fs/staging'),
100
101
    public=os.environ.get('NOMAD_FILES_PUBLIC_DIR', '.volumes/fs/public'),
    prefix_size=int(os.environ.get('NOMAD_FILES_PREFIX_SIZE', 2))
102
)
103
elastic = ElasticConfig(
104
    host=os.environ.get('NOMAD_ELASTIC_HOST', 'localhost'),
105
    port=int(os.environ.get('NOMAD_ELASTIC_PORT', 9200)),
Markus Scheidgen's avatar
Markus Scheidgen committed
106
    index_name=os.environ.get('NOMAD_ELASTIC_INDEX_NAME', 'nomad_fairdi_calcs')
107
)
108
repository_db = RepositoryDBConfig(
109
110
    host=os.environ.get('NOMAD_COE_REPO_DB_HOST', 'localhost'),
    port=int(os.environ.get('NOMAD_COE_REPO_DB_PORT', 5432)),
Markus Scheidgen's avatar
Markus Scheidgen committed
111
    dbname=os.environ.get('NOMAD_COE_REPO_DB_NAME', 'nomad_fairdi_repo_db'),
112
113
    user=os.environ.get('NOMAD_COE_REPO_DB_USER', 'postgres'),
    password=os.environ.get('NOMAD_COE_REPO_PASSWORD', 'nomad')
114
)
115
mongo = MongoConfig(
116
    host=os.environ.get('NOMAD_MONGO_HOST', 'localhost'),
Markus Scheidgen's avatar
Markus Scheidgen committed
117
    port=int(os.environ.get('NOMAD_MONGO_PORT', 27017)),
Markus Scheidgen's avatar
Markus Scheidgen committed
118
    db_name=os.environ.get('NOMAD_MONGO_DB_NAME', 'nomad_fairdi')
119
)
120
logstash = LogstashConfig(
121
    enabled=True,
122
    host=os.environ.get('NOMAD_LOGSTASH_HOST', 'localhost'),
Markus Scheidgen's avatar
Markus Scheidgen committed
123
    tcp_port=int(os.environ.get('NOMAD_LOGSTASH_TCPPORT', '5000')),
124
    level=get_loglevel_from_env('NOMAD_LOGSTASH_LEVEL', default_level=logging.DEBUG)
125
)
Markus Scheidgen's avatar
Markus Scheidgen committed
126
services = NomadServicesConfig(
127
128
    api_host=os.environ.get('NOMAD_API_HOST', 'localhost'),
    api_port=int(os.environ.get('NOMAD_API_PORT', 8000)),
129
    api_base_path=os.environ.get('NOMAD_API_BASE_PATH', '/nomad/api'),
130
131
    api_secret=os.environ.get('NOMAD_API_SECRET', 'defaultApiSecret'),
    admin_password=os.environ.get('NOMAD_API_ADMIN_PASSWORD', 'password'),
132
    upload_url=os.environ.get('NOMAD_UPLOAD_URL', 'http://localhost/nomad/uploads'),
133
    disable_reset=os.environ.get('NOMAD_API_DISABLE_RESET', 'false') == 'true'
Markus Scheidgen's avatar
Markus Scheidgen committed
134
)
135
136
137
138
139
140
141
migration_source_db = RepositoryDBConfig(
    host=os.environ.get('NOMAD_MIGRATION_SOURCE_DB_HOST', 'db-repository.nomad.esc'),
    port=int(os.environ.get('NOMAD_MIGRATION_SOURCE_DB_PORT', 5432)),
    dbname=os.environ.get('NOMAD_MIGRATION_SOURCE_DB_NAME', 'nomad_prod'),
    user=os.environ.get('NOMAD_MIGRATION_SOURCE_USER', 'nomadlab'),
    password=os.environ.get('NOMAD_MIGRATION_SOURCE_PASSWORD', '*')
)
142
mail = MailConfig(
143
    host=os.environ.get('NOMAD_SMTP_HOST', ''),  # empty or None host disables email
144
145
146
147
148
    port=int(os.environ.get('NOMAD_SMTP_PORT', 8995)),
    user=os.environ.get('NOMAD_SMTP_USER', None),
    password=os.environ.get('NOMAD_SMTP_PASSWORD', None),
    from_address=os.environ.get('NOMAD_MAIL_FROM', 'webmaster@nomad-coe.eu')
)
Markus Scheidgen's avatar
Markus Scheidgen committed
149
150
151
normalize = NormalizeConfig(
    all_systems=False
)
Markus Scheidgen's avatar
Markus Scheidgen committed
152

Markus Scheidgen's avatar
Markus Scheidgen committed
153
console_log_level = get_loglevel_from_env('NOMAD_CONSOLE_LOGLEVEL', default_level=logging.WARNING)
154
service = os.environ.get('NOMAD_SERVICE', 'unknown nomad service')
155
release = os.environ.get('NOMAD_RELEASE', 'devel')
Markus Scheidgen's avatar
Markus Scheidgen committed
156
157
158
159
160
161

auxfile_cutoff = 30
"""
Number of max auxfiles. More auxfiles means no auxfiles, but probably a directory of many
mainfiles.
"""