config.py 7.65 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

Markus Scheidgen's avatar
Markus Scheidgen committed
15
import logging
16
17
18
import os
import os.path
import yaml
Markus Scheidgen's avatar
Markus Scheidgen committed
19
20
21
22
23
import warnings

warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

24
25
from nomad import gitinfo

26

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
class NomadConfig(dict):
    """
    A dict subclass that uses attributes as key/value pairs.
    """
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def __getattr__(self, name):
        if name in self:
            return self[name]
        else:
            raise AttributeError("No such attribute: " + name)

    def __setattr__(self, name, value):
        self[name] = value
42

43
44
45
46
47
    def __delattr__(self, name):
        if name in self:
            del self[name]
        else:
            raise AttributeError("No such attribute: " + name)
48
49


50
51
CELERY_WORKER_ROUTING = 'worker'
CELERY_QUEUE_ROUTING = 'queue'
52

53
54
55
56
57
rabbitmq = NomadConfig(
    host='localhost',
    user='rabbitmq',
    password='rabbitmq'
)
58

59

60
61
def rabbitmq_url():
    return 'pyamqp://%s:%s@%s//' % (rabbitmq.user, rabbitmq.password, rabbitmq.host)
62

63

64
65
66
67
celery = NomadConfig(
    max_memory=64e6,  # 64 GB
    timeout=1800,  # 1/2 h
    acks_late=True,
68
    routing=CELERY_QUEUE_ROUTING,
69
70
71
72
73
    priorities={
        'Upload.process_upload': 5,
        'Upload.delete_upload': 9,
        'Upload.publish_upload': 10
    }
74
)
75

76
77
78
79
fs = NomadConfig(
    tmp='.volumes/fs/tmp',
    staging='.volumes/fs/staging',
    public='.volumes/fs/public',
80
    migration_packages='.volumes/fs/migration_packages',
81
    local_tmp='/tmp',
82
83
    prefix_size=2,
    working_directory=os.getcwd()
84
)
85

86
87
88
89
90
elastic = NomadConfig(
    host='localhost',
    port=9200,
    index_name='nomad_fairdi_calcs'
)
91

92
repository_db = NomadConfig(
93
94
    sequential_publish=False,
    publish_enabled=True,
95
96
97
98
99
100
    host='localhost',
    port=5432,
    dbname='nomad_fairdi_repo_db',
    user='postgres',
    password='nomad'
)
Markus Scheidgen's avatar
Markus Scheidgen committed
101

102
103
104
105
106
mongo = NomadConfig(
    host='localhost',
    port=27017,
    db_name='nomad_fairdi'
)
107

108
109
110
111
112
113
logstash = NomadConfig(
    enabled=True,
    host='localhost',
    tcp_port='5000',
    level=logging.DEBUG
)
Markus Scheidgen's avatar
Markus Scheidgen committed
114

115
116
117
118
119
120
services = NomadConfig(
    api_host='localhost',
    api_port=8000,
    api_base_path='/nomad/api',
    api_secret='defaultApiSecret',
    admin_password='password',
121
    disable_reset=True,
122
    not_processed_value='not processed',
123
    unavailable_value='unavailable',
Markus Scheidgen's avatar
Markus Scheidgen committed
124
    https=False
125
126
)

Markus Scheidgen's avatar
Markus Scheidgen committed
127
128
129
130
tests = NomadConfig(
    default_timeout=30
)

131

Markus Scheidgen's avatar
Markus Scheidgen committed
132
133
134
135
136
137
def api_url():
    return '%s://%s%s/%s' % (
        'https' if services.https else 'http',
        services.api_host,
        ':%s' % services.api_port if services.api_port != 80 else '',
        services.api_base_path)
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167


migration_source_db = NomadConfig(
    host='db-repository.nomad.esc',
    port=5432,
    dbname='nomad_prod',
    user='nomadlab',
    password='*'
)

mail = NomadConfig(
    enabled=False,
    with_login=False,
    host='',
    port=8995,
    user='',
    password='',
    from_address='webmaster@nomad-coe.eu'
)

normalize = NomadConfig(
    all_systems=False
)

client = NomadConfig(
    user='leonard.hofstadter@nomad-fairdi.tests.de',
    password='password',
    url='http://localhost:8000/nomad/api'
)

168
version = '0.4.4'
169
commit = gitinfo.commit
170
release = 'devel'
171
172
domain = 'DFT'
service = 'unknown nomad service'
173
auxfile_cutoff = 30
174
console_log_level = logging.WARNING
175

176

177
178
def normalize_loglevel(value, default_level=logging.INFO):
    plain_value = value
179
180
181
182
183
184
    if plain_value is None:
        return default_level
    else:
        try:
            return int(plain_value)
        except ValueError:
185
            return getattr(logging, plain_value)
186
187


188
transformations = {
189
190
    'console_log_level': normalize_loglevel,
    'logstash_level': normalize_loglevel
191
}
Markus Scheidgen's avatar
Markus Scheidgen committed
192

Markus Scheidgen's avatar
Markus Scheidgen committed
193

194
195
196
197
# use std python logger, since logging is not configured while loading configuration
logger = logging.getLogger(__name__)


198
def apply(key, value) -> None:
199
200
201
202
203
204
205
206
    """
    Changes the config according to given key and value. The keys are interpreted as paths
    to config values with ``_`` as a separator. E.g. ``fs_staging`` leading to
    ``config.fs.staging``
    """
    path = list(reversed(key.split('_')))
    child_segment = None
    current_value = None
207
    child_config = globals()
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
    child_key = None

    try:
        while len(path) > 0:
            if child_segment is None:
                child_segment = path.pop()
            else:
                child_segment += '_' + path.pop()

            if child_segment in child_config:
                current_value = child_config[child_segment]

            if current_value is None:
                if len(path) == 0:
                    raise KeyError
223
224

                continue
225
226
            if isinstance(current_value, NomadConfig):
                child_config = current_value
227
                current_value = None
228
229
230
231
                child_segment = None
            else:
                if len(path) > 0:
                    raise KeyError()
Markus Scheidgen's avatar
Markus Scheidgen committed
232
233
234

                child_key = child_segment
                break
235
236
237
238

        if child_key is None or current_value is None:
            raise KeyError()
    except KeyError:
Markus Scheidgen's avatar
Markus Scheidgen committed
239
        return
240
241
242
243
244
245
246
247
248
249
250

    if not isinstance(value, type(current_value)):
        try:
            value = transformations.get(key, type(current_value))(value)
        except Exception as e:
            logger.error(
                'config key %s value %s has wrong type: %s' % (key, str(value), str(e)))

    child_config[child_key] = value


Markus Scheidgen's avatar
Markus Scheidgen committed
251
252
def load_config(config_file: str = os.environ.get('NOMAD_CONFIG', 'nomad.yaml')) -> None:
    # load yaml and override defaults
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
    if os.path.exists(config_file):
        with open(config_file, 'r') as stream:
            try:
                config_data = yaml.load(stream)
            except yaml.YAMLError as e:
                logger.error('cannot read nomad config', exc_info=e)

        def adapt(config, new_config, child_key=None):
            for key, value in new_config.items():
                if key in config:
                    if child_key is None:
                        qualified_key = key
                    else:
                        qualified_key = '%s_%s' % (child_key, key)

                    current_value = config[key]
                    if isinstance(value, dict) and isinstance(current_value, NomadConfig):
                        adapt(current_value, value, qualified_key)
                    else:
                        if not isinstance(value, type(current_value)):
                            try:
                                value = transformations.get(qualified_key, type(current_value))(value)
                            except Exception as e:
                                logger.error(
                                    'config key %s value %s has wrong type: %s' % (key, str(value), str(e)))
                        else:
                            config[key] = value
                else:
                    logger.error('config key %s does not exist' % key)

        adapt(globals(), config_data)

Markus Scheidgen's avatar
Markus Scheidgen committed
285
    # load env and override yaml and defaults
286
287
288
289
290
    kwargs = {
        key[len('NOMAD_'):].lower(): value
        for key, value in os.environ.items()
        if key.startswith('NOMAD_')
    }
291

292
    for key, value in kwargs.items():
293
        apply(key, value)
294
295
296


load_config()