config.py 6.3 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
17
18
19
"""
This module is used to store all configuration values. It makes use of
*namedtuples* to create key sensitive configuration objects.
"""

20
import os
Markus Scheidgen's avatar
Markus Scheidgen committed
21
import logging
22
from collections import namedtuple
Markus Scheidgen's avatar
Markus Scheidgen committed
23
24
25
26
27
import warnings

warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

28

29
FilesConfig = namedtuple(
30
    'FilesConfig', ['uploads_bucket', 'raw_bucket', 'archive_bucket', 'staging_bucket', 'public_bucket'])
31
32
""" API independent configuration for the object storage. """

33
CeleryConfig = namedtuple('Celery', ['broker_url', 'max_memory', 'timeout', 'acks_late'])
Markus Scheidgen's avatar
Markus Scheidgen committed
34
""" Used to configure the RabbitMQ for celery. """
35

36
FSConfig = namedtuple('FSConfig', ['tmp', 'objects', 'nomad_tmp'])
37
38
""" Used to configure file stystem access. """

39
40
41
RepositoryDBConfig = namedtuple('RepositoryDBConfig', ['host', 'port', 'dbname', 'user', 'password'])
""" Used to configure access to NOMAD-coe repository db. """

Markus Scheidgen's avatar
Markus Scheidgen committed
42
ElasticConfig = namedtuple('ElasticConfig', ['host', 'port', 'index_name'])
43
44
""" Used to configure elastic search. """

45
MongoConfig = namedtuple('MongoConfig', ['host', 'port', 'db_name'])
46
47
""" Used to configure mongo db. """

Markus Scheidgen's avatar
Markus Scheidgen committed
48
LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port', 'level'])
49
""" Used to configure and enable/disable the ELK based centralized logging. """
50

51
NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_host', 'api_port', 'api_base_path', 'api_secret', 'admin_password', 'upload_url', 'disable_reset'])
Markus Scheidgen's avatar
Markus Scheidgen committed
52
53
""" Used to configure nomad services: worker, handler, api """

54
MailConfig = namedtuple('MailConfig', ['host', 'port', 'user', 'password', 'from_address'])
55
56
""" Used to configure how nomad can send email """

Markus Scheidgen's avatar
Markus Scheidgen committed
57
58
59
NormalizeConfig = namedtuple('NormalizeConfig', ['all_systems'])
""" Used to configure the normalizers """

60
files = FilesConfig(
61
    uploads_bucket='uploads',
62
    raw_bucket=os.environ.get('NOMAD_FILES_RAW_BUCKET', 'raw'),
63
    archive_bucket='archive',
64
65
    staging_bucket='staging',
    public_bucket='public'
66
)
67
68
69
70
71
72
73

rabbit_host = os.environ.get('NOMAD_RABBITMQ_HOST', 'localhost')
rabbit_port = os.environ.get('NOMAD_RABBITMQ_PORT', None)
rabbit_user = 'rabbitmq'
rabbit_password = 'rabbitmq'
rabbit_url = 'pyamqp://%s:%s@%s//' % (rabbit_user, rabbit_password, rabbit_host)

74
75
76
77
78
79
80
81
82
83
84
85

def get_loglevel_from_env(key, default_level=logging.INFO):
    plain_value = os.environ.get(key, None)
    if plain_value is None:
        return default_level
    else:
        try:
            return int(plain_value)
        except ValueError:
            return getattr(logging, plain_value, default_level)


86
celery = CeleryConfig(
87
88
    broker_url=rabbit_url,
    max_memory=int(os.environ.get('NOMAD_CELERY_MAXMEMORY', 64e6)),  # 64 GB
89
90
    timeout=int(os.environ.get('NOMAD_CELERY_TIMEOUT', 3 * 3600)),  # 3h
    acks_late=bool(os.environ.get('NOMAD_CELERY_ACKS_LATE', True))
91
)
92

93
fs = FSConfig(
94
    tmp=os.environ.get('NOMAD_FILES_TMP_DIR', '.volumes/fs/tmp'),
95
96
    objects=os.environ.get('NOMAD_FILES_OBJECTS_DIR', '.volumes/fs/objects'),
    nomad_tmp=os.environ.get('NOMAD_FILES_NOMAD_TMP_DIR', '/nomad/tmp')
97
)
98
elastic = ElasticConfig(
99
    host=os.environ.get('NOMAD_ELASTIC_HOST', 'localhost'),
100
    port=int(os.environ.get('NOMAD_ELASTIC_PORT', 9200)),
Markus Scheidgen's avatar
Markus Scheidgen committed
101
    index_name=os.environ.get('NOMAD_ELASTIC_INDEX_NAME', 'nomad_fairdi_calcs')
102
)
103
repository_db = RepositoryDBConfig(
104
105
    host=os.environ.get('NOMAD_COE_REPO_DB_HOST', 'localhost'),
    port=int(os.environ.get('NOMAD_COE_REPO_DB_PORT', 5432)),
Markus Scheidgen's avatar
Markus Scheidgen committed
106
    dbname=os.environ.get('NOMAD_COE_REPO_DB_NAME', 'nomad_fairdi_repo_db'),
107
108
    user=os.environ.get('NOMAD_COE_REPO_DB_USER', 'postgres'),
    password=os.environ.get('NOMAD_COE_REPO_PASSWORD', 'nomad')
109
)
110
mongo = MongoConfig(
111
    host=os.environ.get('NOMAD_MONGO_HOST', 'localhost'),
Markus Scheidgen's avatar
Markus Scheidgen committed
112
    port=int(os.environ.get('NOMAD_MONGO_PORT', 27017)),
Markus Scheidgen's avatar
Markus Scheidgen committed
113
    db_name=os.environ.get('NOMAD_MONGO_DB_NAME', 'nomad_fairdi')
114
)
115
logstash = LogstashConfig(
116
    enabled=True,
117
    host=os.environ.get('NOMAD_LOGSTASH_HOST', 'localhost'),
Markus Scheidgen's avatar
Markus Scheidgen committed
118
    tcp_port=int(os.environ.get('NOMAD_LOGSTASH_TCPPORT', '5000')),
119
    level=get_loglevel_from_env('NOMAD_LOGSTASH_LEVEL', default_level=logging.DEBUG)
120
)
Markus Scheidgen's avatar
Markus Scheidgen committed
121
services = NomadServicesConfig(
122
123
    api_host=os.environ.get('NOMAD_API_HOST', 'localhost'),
    api_port=int(os.environ.get('NOMAD_API_PORT', 8000)),
124
    api_base_path=os.environ.get('NOMAD_API_BASE_PATH', '/nomad/api'),
125
126
    api_secret=os.environ.get('NOMAD_API_SECRET', 'defaultApiSecret'),
    admin_password=os.environ.get('NOMAD_API_ADMIN_PASSWORD', 'password'),
127
    upload_url=os.environ.get('NOMAD_UPLOAD_URL', 'http://localhost/nomad/uploads'),
128
    disable_reset=os.environ.get('NOMAD_API_DISABLE_RESET', 'false') == 'true'
Markus Scheidgen's avatar
Markus Scheidgen committed
129
)
130
131
132
133
134
135
136
migration_source_db = RepositoryDBConfig(
    host=os.environ.get('NOMAD_MIGRATION_SOURCE_DB_HOST', 'db-repository.nomad.esc'),
    port=int(os.environ.get('NOMAD_MIGRATION_SOURCE_DB_PORT', 5432)),
    dbname=os.environ.get('NOMAD_MIGRATION_SOURCE_DB_NAME', 'nomad_prod'),
    user=os.environ.get('NOMAD_MIGRATION_SOURCE_USER', 'nomadlab'),
    password=os.environ.get('NOMAD_MIGRATION_SOURCE_PASSWORD', '*')
)
137
mail = MailConfig(
138
    host=os.environ.get('NOMAD_SMTP_HOST', ''),  # empty or None host disables email
139
140
141
142
143
    port=int(os.environ.get('NOMAD_SMTP_PORT', 8995)),
    user=os.environ.get('NOMAD_SMTP_USER', None),
    password=os.environ.get('NOMAD_SMTP_PASSWORD', None),
    from_address=os.environ.get('NOMAD_MAIL_FROM', 'webmaster@nomad-coe.eu')
)
Markus Scheidgen's avatar
Markus Scheidgen committed
144
145
146
normalize = NormalizeConfig(
    all_systems=False
)
Markus Scheidgen's avatar
Markus Scheidgen committed
147

Markus Scheidgen's avatar
Markus Scheidgen committed
148
console_log_level = get_loglevel_from_env('NOMAD_CONSOLE_LOGLEVEL', default_level=logging.WARNING)
149
service = os.environ.get('NOMAD_SERVICE', 'unknown nomad service')
150
release = os.environ.get('NOMAD_RELEASE', 'devel')
Markus Scheidgen's avatar
Markus Scheidgen committed
151
152
153
154
155
156

auxfile_cutoff = 30
"""
Number of max auxfiles. More auxfiles means no auxfiles, but probably a directory of many
mainfiles.
"""