repo.py 27.9 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
The repository API of the nomad@FAIRDI APIs. Currently allows to resolve repository
meta-data.
"""

20
from typing import List, Dict, Any
Markus Scheidgen's avatar
Markus Scheidgen committed
21
from flask_restplus import Resource, abort, fields
22
from flask import request, g
23
from elasticsearch.exceptions import NotFoundError
24
import elasticsearch.helpers
Markus Scheidgen's avatar
Markus Scheidgen committed
25

26
27
from nomad import search, utils, datamodel, processing as proc, infrastructure
from nomad.app.utils import rfc3339DateTime, RFC3339DateTime, with_logger
28
from nomad.app.optimade import filterparser
29
from nomad.datamodel import UserMetadata, Dataset, User
Markus Scheidgen's avatar
Markus Scheidgen committed
30

Markus Scheidgen's avatar
Markus Scheidgen committed
31
from .api import api
32
from .auth import authenticate
33
from .common import pagination_model, pagination_request_parser, calc_route
Markus Scheidgen's avatar
Markus Scheidgen committed
34

35
ns = api.namespace('repo', description='Access repository metadata.')
Markus Scheidgen's avatar
Markus Scheidgen committed
36
37
38
39
40


@calc_route(ns)
class RepoCalcResource(Resource):
    @api.response(404, 'The upload or calculation does not exist')
41
    @api.response(401, 'Not authorized to access the calculation')
42
    @api.response(200, 'Metadata send', fields.Raw)
43
    @api.doc('get_repo_calc')
44
    @authenticate()
45
    def get(self, upload_id, calc_id):
Markus Scheidgen's avatar
Markus Scheidgen committed
46
47
48
        """
        Get calculation metadata in repository form.

49
        Repository metadata only entails the quantities shown in the repository.
50
        Calcs are references via *upload_id*, *calc_id* pairs.
Markus Scheidgen's avatar
Markus Scheidgen committed
51
52
        """
        try:
53
54
55
56
57
58
59
60
            calc = search.Entry.get(calc_id)
        except NotFoundError:
            abort(404, message='There is no calculation %s/%s' % (upload_id, calc_id))

        if calc.with_embargo or not calc.published:
            if g.user is None:
                abort(401, message='Not logged in to access %s/%s.' % (upload_id, calc_id))

61
            if not (any(g.user.user_id == user.user_id for user in calc.owners) or g.user.is_admin):
62
63
64
                abort(401, message='Not authorized to access %s/%s.' % (upload_id, calc_id))

        return calc.to_dict(), 200
Markus Scheidgen's avatar
Markus Scheidgen committed
65
66
67


repo_calcs_model = api.model('RepoCalculations', {
Markus Scheidgen's avatar
Markus Scheidgen committed
68
    'pagination': fields.Nested(pagination_model, skip_none=True),
69
70
71
72
    'scroll': fields.Nested(allow_null=True, skip_none=True, model=api.model('Scroll', {
        'total': fields.Integer(description='The total amount of hits for the search.'),
        'scroll_id': fields.String(allow_null=True, description='The scroll_id that can be used to retrieve the next page.'),
        'size': fields.Integer(help='The size of the returned scroll page.')})),
73
74
75
    'results': fields.List(fields.Raw, description=(
        'A list of search results. Each result is a dict with quantitie names as key and '
        'values as values')),
76
77
78
79
    'statistics': fields.Raw(description=(
        'A dict with all statistics. Each statistic is dictionary with a metrics dict as '
        'value and quantity value as key. The possible metrics are code runs(calcs), %s. '
        'There is a pseudo quantity "total" with a single value "all" that contains the '
80
        ' metrics over all results. ' % ', '.join(datamodel.Domain.instance.metrics_names))),
81
    'datasets': fields.Nested(api.model('RepoDatasets', {
82
        'after': fields.String(description='The after value that can be used to retrieve the next datasets.'),
83
84
85
86
87
        'values': fields.Raw(description='A dict with dataset id as key. The values are dicts with "total" and "examples" keys.')
    }), skip_none=True),
    'uploads': fields.Nested(api.model('RepoUploads', {
        'after': fields.String(description='The after value that can be used to retrieve the next uploads.'),
        'values': fields.Raw(description='A dict with upload ids as key. The values are dicts with "total" and "examples" keys.')
Markus Scheidgen's avatar
Markus Scheidgen committed
88
    }), skip_none=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
89
90
})

91

92
93
repo_calc_id_model = api.model('RepoCalculationId', {
    'upload_id': fields.String(), 'calc_id': fields.String()
Markus Scheidgen's avatar
Markus Scheidgen committed
94
95
})

96
97
98
99
100
101
102
103
104
105
106
107

def add_common_parameters(request_parser):
    request_parser.add_argument(
        'owner', type=str,
        help='Specify which calcs to return: ``all``, ``public``, ``user``, ``staging``, default is ``all``')
    request_parser.add_argument(
        'from_time', type=lambda x: rfc3339DateTime.parse(x),
        help='A yyyy-MM-ddTHH:mm:ss (RFC3339) minimum entry time (e.g. upload time)')
    request_parser.add_argument(
        'until_time', type=lambda x: rfc3339DateTime.parse(x),
        help='A yyyy-MM-ddTHH:mm:ss (RFC3339) maximum entry time (e.g. upload time)')

108
    for quantity in search.quantities.values():
109
        request_parser.add_argument(
110
            quantity.name, help=quantity.description,
111
            action=quantity.argparse_action if quantity.multi else None)
112
113


Markus Scheidgen's avatar
Markus Scheidgen committed
114
repo_request_parser = pagination_request_parser.copy()
115
add_common_parameters(repo_request_parser)
116
117
118
119
repo_request_parser.add_argument(
    'scroll', type=bool, help='Enable scrolling')
repo_request_parser.add_argument(
    'scroll_id', type=str, help='The id of the current scrolling window to use.')
120
121
repo_request_parser.add_argument(
    'date_histogram', type=bool, help='Add an additional aggregation over the upload time')
122
repo_request_parser.add_argument(
123
    'datasets_after', type=str, help='The last dataset id of the last scroll window for the dataset quantity')
124
125
repo_request_parser.add_argument(
    'uploads_after', type=str, help='The last upload id of the last scroll window for the upload quantity')
Markus Scheidgen's avatar
Markus Scheidgen committed
126
repo_request_parser.add_argument(
127
    'metrics', type=str, action='append', help=(
128
        'Metrics to aggregate over all quantities and their values as comma separated list. '
129
        'Possible values are %s.' % ', '.join(datamodel.Domain.instance.metrics_names)))
Markus Scheidgen's avatar
Markus Scheidgen committed
130
131
repo_request_parser.add_argument(
    'datasets', type=bool, help=('Return dataset information.'))
132
133
repo_request_parser.add_argument(
    'uploads', type=bool, help=('Return upload information.'))
Markus Scheidgen's avatar
Markus Scheidgen committed
134
135
repo_request_parser.add_argument(
    'statistics', type=bool, help=('Return statistics.'))
Markus Scheidgen's avatar
Markus Scheidgen committed
136

137

138
139
140
141
search_request_parser = api.parser()
add_common_parameters(search_request_parser)


142
def add_query(search_request: search.SearchRequest, args: Dict[str, Any]):
143
    """
144
    Help that adds query relevant request args to the given SearchRequest.
145
    """
146
    args = {key: value for key, value in args.items() if value is not None}
147

148
    # owner
149
    owner = args.get('owner', 'all')
150
151
    try:
        search_request.owner(
152
            owner,
153
154
            g.user.user_id if g.user is not None else None)
    except ValueError as e:
155
        abort(401, getattr(e, 'message', 'Invalid owner parameter: %s' % owner))
156
157
158
159
    except Exception as e:
        abort(400, getattr(e, 'message', 'Invalid owner parameter'))

    # time range
160
161
    from_time_str = args.get('from_time', None)
    until_time_str = args.get('until_time', None)
162
163

    try:
164
165
166
        from_time = rfc3339DateTime.parse(from_time_str) if from_time_str is not None else None
        until_time = rfc3339DateTime.parse(until_time_str) if until_time_str is not None else None
        search_request.time_range(start=from_time, end=until_time)
167
168
169
    except Exception:
        abort(400, message='bad datetime format')

170
171
    # optimade
    try:
172
        optimade = args.get('optimade', None)
173
174
175
176
177
178
        if optimade is not None:
            q = filterparser.parse_filter(optimade)
            search_request.query(q)
    except filterparser.FilterException:
        abort(400, message='could not parse optimade query')

179
180
    # search parameter
    search_request.search_parameters(**{
181
        key: value for key, value in args.items()
182
        if key not in ['optimade'] and key in search.quantities})
183
184


Markus Scheidgen's avatar
Markus Scheidgen committed
185
186
@ns.route('/')
class RepoCalcsResource(Resource):
187
    @api.doc('search')
188
    @api.response(400, 'Invalid requests, e.g. wrong owner type or bad search parameters')
Markus Scheidgen's avatar
Markus Scheidgen committed
189
    @api.expect(repo_request_parser, validate=True)
190
    @api.marshal_with(repo_calcs_model, skip_none=True, code=200, description='Search results send')
191
    @authenticate()
Markus Scheidgen's avatar
Markus Scheidgen committed
192
193
    def get(self):
        """
194
        Search for calculations in the repository form, paginated.
195
196

        The ``owner`` parameter determines the overall entries to search through.
197
198
199
200
        Possible values are: ``all`` (show all entries visible to the current user), ``public``
        (show all publically visible entries), ``user`` (show all user entries, requires login),
        ``staging`` (show all user entries in staging area, requires login).

201
202
203
204
205
        You can use the various quantities to search/filter for. For some of the
        indexed quantities this endpoint returns aggregation information. This means
        you will be given a list of all possible values and the number of entries
        that have the certain value. You can also use these aggregations on an empty
        search to determine the possible values.
206
207
208

        The pagination parameters allows determine which page to return via the
        ``page`` and ``per_page`` parameters. Pagination however, is limited to the first
Markus Scheidgen's avatar
Markus Scheidgen committed
209
210
211
212
213
214
215
216
        100k (depending on ES configuration) hits.

        An alternative to pagination is to use ``scroll`` and ``scroll_id``. With ``scroll``
        you will get a ``scroll_id`` on the first request. Each call with ``scroll`` and
        the respective ``scroll_id`` will return the next ``per_page`` (here the default is 1000)
        results. Scroll however, ignores ordering and does not return aggregations.
        The scroll view used in the background will stay alive for 1 minute between requests.
        If the given ``scroll_id`` is not available anymore, a HTTP 400 is raised.
217
218
219
220
221

        The search will return aggregations on a predefined set of quantities. Aggregations
        will tell you what quantity values exist and how many entries match those values.

        Ordering is determined by ``order_by`` and ``order`` parameters.
222
        """
223
224

        try:
225
226
227
228
229
230
231
232
233
234
235
236
            args = {
                key: value for key, value in repo_request_parser.parse_args().items()
                if value is not None}

            scroll = args.get('scroll', False)
            scroll_id = args.get('scroll_id', None)
            page = args.get('page', 1)
            per_page = args.get('per_page', 10 if not scroll else 1000)
            order = args.get('order', -1)
            order_by = args.get('order_by', 'formula')

            date_histogram = args.get('date_histogram', False)
237
            metrics: List[str] = request.args.getlist('metrics')
Markus Scheidgen's avatar
Markus Scheidgen committed
238

239
            with_datasets = args.get('datasets', False)
240
241
            with_uploads = args.get('uploads', False)
            with_statistics = args.get('statistics', False) or with_datasets or with_uploads
242
243
244
245
246
247
248
        except Exception as e:
            abort(400, message='bad parameters: %s' % str(e))

        search_request = search.SearchRequest()
        add_query(search_request, args)
        if date_histogram:
            search_request.date_histogram()
249

250
        try:
251
            assert page >= 1
252
            assert per_page >= 0
253
254
255
        except AssertionError:
            abort(400, message='invalid pagination')

256
257
258
        if order not in [-1, 1]:
            abort(400, message='invalid pagination')

259
260
        for metric in metrics:
            if metric not in search.metrics_names:
261
262
                abort(400, message='there is no metric %s' % metric)

Markus Scheidgen's avatar
Markus Scheidgen committed
263
264
        if with_statistics:
            search_request.default_statistics(metrics_to_use=metrics)
265
266
267
268
269
270
271
272
273

            additional_metrics = []
            if with_datasets and 'datasets' not in metrics:
                additional_metrics.append('datasets')
            if with_uploads and 'uploads' not in metrics:
                additional_metrics.append('uploads')

            total_metrics = metrics + additional_metrics

Markus Scheidgen's avatar
Markus Scheidgen committed
274
275
            search_request.totals(metrics_to_use=total_metrics)
            search_request.statistic('authors', 1000)
276

277
        try:
278
            if scroll:
279
                results = search_request.execute_scrolled(scroll_id=scroll_id, size=per_page)
280

281
            else:
Markus Scheidgen's avatar
Markus Scheidgen committed
282
283
284
285
286
                if with_datasets:
                    search_request.quantity(
                        'dataset_id', size=per_page, examples=1,
                        after=request.args.get('datasets_after', None))

287
288
289
290
291
                if with_uploads:
                    search_request.quantity(
                        'upload_id', size=per_page, examples=1,
                        after=request.args.get('uploads_after', None))

292
293
                results = search_request.execute_paginated(
                    per_page=per_page, page=page, order=order, order_by=order_by)
294
295

                # TODO just a work around to make things prettier
Markus Scheidgen's avatar
Markus Scheidgen committed
296
297
298
299
300
                if with_statistics:
                    statistics = results['statistics']
                    if 'code_name' in statistics and 'currupted mainfile' in statistics['code_name']:
                        del(statistics['code_name']['currupted mainfile'])

301
302
303
                if 'quantities' in results:
                    quantities = results.pop('quantities')

Markus Scheidgen's avatar
Markus Scheidgen committed
304
                if with_datasets:
305
                    datasets = quantities['dataset_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
306
                    results['datasets'] = datasets
307

308
309
310
311
                if with_uploads:
                    uploads = quantities['upload_id']
                    results['uploads'] = uploads

312
            return results, 200
Markus Scheidgen's avatar
Markus Scheidgen committed
313
314
        except search.ScrollIdNotFound:
            abort(400, 'The given scroll_id does not exist.')
315
        except KeyError as e:
316
317
            import traceback
            traceback.print_exc()
318
            abort(400, str(e))
319

320
321
322
323
324
325
326
327

query_model_parameters = {
    'owner': fields.String(description='Specify which calcs to return: ``all``, ``public``, ``user``, ``staging``, default is ``all``'),
    'from_time': RFC3339DateTime(description='A yyyy-MM-ddTHH:mm:ss (RFC3339) minimum entry time (e.g. upload time)'),
    'until_time': RFC3339DateTime(description='A yyyy-MM-ddTHH:mm:ss (RFC3339) maximum entry time (e.g. upload time)')
}

for quantity in search.quantities.values():
328
    if quantity.multi and quantity.argparse_action is None:
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
        def field(**kwargs):
            return fields.List(fields.String(**kwargs))
    else:
        field = fields.String
    query_model_parameters[quantity.name] = field(description=quantity.description)

repo_query_model = api.model('RepoQuery', query_model_parameters, skip_none=True)


def repo_edit_action_field(quantity):
    if quantity.is_scalar:
        return fields.Nested(repo_edit_action_model, description=quantity.description, skip_none=True)
    else:
        return fields.List(
            fields.Nested(repo_edit_action_model, skip_none=True), description=quantity.description)


repo_edit_action_model = api.model('RepoEditAction', {
    'value': fields.String(description='The value/values that is set as a string.'),
    'success': fields.Boolean(description='If this can/could be done. Only in API response.'),
    'message': fields.String(descriptin='A message that details the action result. Only in API response.')
})

repo_edit_model = api.model('RepoEdit', {
    'verify': fields.Boolean(description='If true, no action is performed.'),
    'query': fields.Nested(repo_query_model, skip_none=True, description='New metadata will be applied to query results.'),
    'actions': fields.Nested(
        api.model('RepoEditActions', {
            quantity.name: repo_edit_action_field(quantity)
            for quantity in UserMetadata.m_def.all_quantities.values()
        }), skip_none=True,
        description='Each action specifies a single value (even for multi valued quantities).')
})


364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
def edit(parsed_query: Dict[str, Any], logger, mongo_update: Dict[str, Any] = None, re_index=True):
    # get all calculations that have to change
    search_request = search.SearchRequest()
    add_query(search_request, parsed_query)
    calc_ids = list(hit['calc_id'] for hit in search_request.execute_scan())

    # perform the update on the mongo db
    if mongo_update is not None:
        n_updated = proc.Calc.objects(calc_id__in=calc_ids).update(multi=True, **mongo_update)
        if n_updated != len(calc_ids):
            logger.error('edit repo did not update all entries', payload=mongo_update)

    # re-index the affected entries in elastic search
    if re_index:
        def elastic_updates():
            for calc in proc.Calc.objects(calc_id__in=calc_ids):
                entry = search.Entry.from_calc_with_metadata(
                    datamodel.CalcWithMetadata(**calc['metadata']))
                entry = entry.to_dict(include_meta=True)
                entry['_op_type'] = 'index'
                yield entry

        _, failed = elasticsearch.helpers.bulk(
            infrastructure.elastic_client, elastic_updates(), stats_only=True)
        search.refresh()
        if failed > 0:
            logger.error(
                'edit repo with failed elastic updates',
                payload=mongo_update, nfailed=len(failed))


395
396
@ns.route('/edit')
class EditRepoCalcsResource(Resource):
397
398
399
    @api.doc('edit_repo')
    @api.response(400, 'Invalid requests, e.g. wrong owner type or bad search parameters')
    @api.expect(repo_edit_model)
400
    @api.marshal_with(repo_edit_model, skip_none=True, code=200, description='Edit verified/performed')
401
402
403
404
    @authenticate()
    @with_logger
    def post(self, logger):
        """ Edit repository metadata. """
405
406

        # basic body parsing and some semantic checks
407
408
409
410
411
412
413
414
415
416
        json_data = request.get_json()
        if json_data is None:
            json_data = {}
        query = json_data.get('query', {})

        owner = query.get('owner', 'user')
        if owner not in ['user', 'staging']:
            abort(400, 'Not a valid owner for edit %s. Edit can only be performed in user or staging' % owner)
        query['owner'] = owner

417
418
419
420
        if 'actions' not in json_data:
            abort(400, 'Missing key actions in edit data')
        actions = json_data['actions']
        verify = json_data.get('verify', False)
421

422
        # checking the edit actions and preparing a mongo update on the fly
423
        mongo_update = {}
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
        for action_quantity_name, quantity_actions in actions.items():
            quantity = UserMetadata.m_def.all_quantities.get(action_quantity_name)
            if quantity is None:
                abort(400, 'Unknown quantity %s' % action_quantity_name)

            quantity_flask = quantity.m_x('flask', {})
            if quantity_flask.get('admin_only', False):
                if not g.user.is_admin():
                    abort(404, 'Only the admin user can set %s' % quantity.name)

            if quantity.name == 'Embargo':
                abort(400, 'Cannot raise an embargo, you can only lift the embargo')

            if isinstance(quantity_actions, list) == quantity.is_scalar:
                abort(400, 'Wrong shape for quantity %s' % action_quantity_name)

            if not isinstance(quantity_actions, list):
                quantity_actions = [quantity_actions]

            flask_verify = quantity_flask.get('verify', None)
            has_error = False
            for action in quantity_actions:
                action['success'] = True
                action['message'] = None
                action_value = action['value'].strip()
                if action_value == '':
                    mongo_value = None

                elif flask_verify == datamodel.User:
                    try:
                        mongo_value = User.get(email=action_value).user_id
                    except KeyError:
                        action['success'] = False
                        has_error = True
                        action['message'] = 'User does not exist'
                        continue

                elif flask_verify == datamodel.Dataset:
                    try:
                        mongo_value = Dataset.m_def.m_x('me').get(
                            user_id=g.user.user_id, name=action_value).dataset_id
                    except KeyError:
                        action['message'] = 'Dataset does not exist and will be created'
                        mongo_value = None
                        if not verify:
                            dataset = Dataset(
                                dataset_id=utils.create_uuid(), user_id=g.user.user_id,
                                name=action_value)
                            dataset.m_x('me').create()
                            mongo_value = dataset.dataset_id

                else:
                    mongo_value = action_value

                mongo_key = 'metadata__%s' % quantity.name
                if len(quantity.shape) == 0:
                    mongo_update[mongo_key] = mongo_value
                else:
                    mongo_values = mongo_update.setdefault(mongo_key, [])
                    if mongo_value is not None:
                        mongo_values.append(mongo_value)

        # stop here, if client just wants to verify its actions
        if verify:
            return json_data, 200

        # stop if the action were not ok
        if has_error:
            return json_data, 400

        # get all calculations that have to change
495
496
497
498
499
500
501
        parsed_query = {}
        for quantity_name, quantity in search.quantities.items():
            if quantity_name in query:
                value = query[quantity_name]
                if quantity.multi and quantity.argparse_action == 'split' and not isinstance(value, list):
                    value = value.split(',')
                parsed_query[quantity_name] = value
502
        parsed_query['owner'] = owner
503

504
505
        # perform the change
        edit(parsed_query, logger, mongo_update, True)
506

507
        return json_data, 200
508

509

510
511
512
513
514
repo_quantity_model = api.model('RepoQuantity', {
    'after': fields.String(description='The after value that can be used to retrieve the next set of values.'),
    'values': fields.Raw(description='A dict with values as key. Values are dicts with "total" and "examples" keys.')
})

515
repo_quantity_values_model = api.model('RepoQuantityValues', {
516
517
518
519
520
    'quantity': fields.Nested(repo_quantity_model, allow_null=True)
})

repo_quantities_model = api.model('RepoQuantities', {
    'quantities': fields.List(fields.Nested(repo_quantity_model))
521
522
523
524
525
526
})

repo_quantity_search_request_parser = api.parser()
add_common_parameters(repo_quantity_search_request_parser)
repo_quantity_search_request_parser.add_argument(
    'after', type=str, help='The after value to use for "scrolling".')
527
repo_quantity_search_request_parser.add_argument(
528
529
530
    'size', type=int, help='The max size of the returned values.')


531
@ns.route('/quantity/<string:quantity>')
532
533
534
535
536
class RepoQuantityResource(Resource):
    @api.doc('quantity_search')
    @api.response(400, 'Invalid requests, e.g. wrong owner type, bad quantity, bad search parameters')
    @api.expect(repo_quantity_search_request_parser, validate=True)
    @api.marshal_with(repo_quantity_values_model, skip_none=True, code=200, description='Search results send')
537
    @authenticate()
538
539
540
541
542
543
544
545
546
547
548
549
550
551
    def get(self, quantity: str):
        """
        Retrieve quantity values from entries matching the search.

        You can use the various quantities to search/filter for. For some of the
        indexed quantities this endpoint returns aggregation information. This means
        you will be given a list of all possible values and the number of entries
        that have the certain value. You can also use these aggregations on an empty
        search to determine the possible values.

        There is no ordering and no pagination. Instead there is an 'after' key based
        scrolling. The result will contain an 'after' value, that can be specified
        for the next request. You can use the 'size' and 'after' parameters accordingly.

552
553
554
        The result will contain a 'quantity' key with quantity values and the "after"
        value. There will be upto 'size' many values. For the rest of the values use the
        "after" parameter in another request.
555
556
        """

557
        search_request = search.SearchRequest()
558
559
560
561
        args = {
            key: value
            for key, value in repo_quantity_search_request_parser.parse_args().items()
            if value is not None}
562

563
564
565
        add_query(search_request, args)
        after = args.get('after', None)
        size = args.get('size', 100)
566
567
568
569
570
571

        try:
            assert size >= 0
        except AssertionError:
            abort(400, message='invalid size')

572
        search_request.quantity(quantity, size=size, after=after)
573
574

        try:
575
576
577
            results = search_request.execute()
            quantities = results.pop('quantities')
            results['quantity'] = quantities[quantity]
578
579
580
581
582
583

            return results, 200
        except KeyError as e:
            import traceback
            traceback.print_exc()
            abort(400, 'Given quantity does not exist: %s' % str(e))
584
585


586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
repo_quantities_search_request_parser = api.parser()
add_common_parameters(repo_quantities_search_request_parser)
repo_quantities_search_request_parser.add_argument(
    'quantities', type=str, action='append',
    help='The quantities to retrieve values from')
repo_quantities_search_request_parser.add_argument(
    'size', type=int, help='The max size of the returned values.')


@ns.route('/quantities')
class RepoQuantitiesResource(Resource):
    @api.doc('quantities_search')
    @api.response(400, 'Invalid requests, e.g. wrong owner type, bad quantity, bad search parameters')
    @api.expect(repo_quantities_search_request_parser, validate=True)
    @api.marshal_with(repo_quantities_model, skip_none=True, code=200, description='Search results send')
    @authenticate()
    def get(self):
        """
        Retrieve quantity values for multiple quantities at once.

        You can use the various quantities to search/filter for. For some of the
        indexed quantities this endpoint returns aggregation information. This means
        you will be given a list of all possible values and the number of entries
        that have the certain value. You can also use these aggregations on an empty
        search to determine the possible values.

        There is no ordering and no pagination and not after key based scrolling. Instead
        there is an 'after' key based scrolling.

        The result will contain a 'quantities' key with a dict of quantity names and the
        retrieved values as values.
        """

        search_request = search.SearchRequest()
        args = {
            key: value
            for key, value in repo_quantities_search_request_parser.parse_args().items()
            if value is not None}

        add_query(search_request, args)
        quantities = args.get('quantities', [])
        size = args.get('size', 5)

        print('A ', quantities)
        try:
            assert size >= 0
        except AssertionError:
            abort(400, message='invalid size')

        for quantity in quantities:
            try:
                search_request.quantity(quantity, size=size)
            except KeyError as e:
                import traceback
                traceback.print_exc()
                abort(400, 'Given quantity does not exist: %s' % str(e))

        return search_request.execute(), 200


646
647
648
649
650
@ns.route('/pid/<int:pid>')
class RepoPidResource(Resource):
    @api.doc('resolve_pid')
    @api.response(404, 'Entry with PID does not exist')
    @api.marshal_with(repo_calc_id_model, skip_none=True, code=200, description='Entry resolved')
Markus Scheidgen's avatar
Markus Scheidgen committed
651
    @authenticate()
652
    def get(self, pid: int):
653
654
655
656
        search_request = search.SearchRequest()

        if g.user is not None:
            search_request.owner('all', user_id=g.user.user_id)
657
        else:
658
659
660
661
662
663
664
665
666
667
668
            search_request.owner('all')

        search_request.search_parameter('pid', pid)

        results = list(search_request.execute_scan())
        total = len(results)

        if total == 0:
            abort(404, 'Entry with PID %d does not exist' % pid)

        if total > 1:
669
            utils.get_logger(__name__).error('Two entries for the same pid', pid=pid)
670
671
672
673
674

        result = results[0]
        return dict(
            upload_id=result['upload_id'],
            calc_id=result['calc_id'])