repo.py 9.13 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
The repository API of the nomad@FAIRDI APIs. Currently allows to resolve repository
meta-data.
"""

from flask_restplus import Resource, abort, fields
21
22
from flask import request, g
from elasticsearch_dsl import Q
23
from elasticsearch.exceptions import NotFoundError
Markus Scheidgen's avatar
Markus Scheidgen committed
24

25
from nomad import search
Markus Scheidgen's avatar
Markus Scheidgen committed
26
27

from .app import api
28
from .auth import login_if_available
Markus Scheidgen's avatar
Markus Scheidgen committed
29
30
from .common import pagination_model, pagination_request_parser, calc_route

31
ns = api.namespace('repo', description='Access repository metadata.')
Markus Scheidgen's avatar
Markus Scheidgen committed
32
33
34
35
36


@calc_route(ns)
class RepoCalcResource(Resource):
    @api.response(404, 'The upload or calculation does not exist')
37
    @api.response(401, 'Not authorized to access the calculation')
38
    @api.response(200, 'Metadata send', fields.Raw)
39
    @api.doc('get_repo_calc')
40
    @login_if_available
41
    def get(self, upload_id, calc_id):
Markus Scheidgen's avatar
Markus Scheidgen committed
42
43
44
        """
        Get calculation metadata in repository form.

45
        Repository metadata only entails the quantities shown in the repository.
46
        Calcs are references via *upload_id*, *calc_id* pairs.
Markus Scheidgen's avatar
Markus Scheidgen committed
47
48
        """
        try:
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
            calc = search.Entry.get(calc_id)
        except NotFoundError:
            abort(404, message='There is no calculation %s/%s' % (upload_id, calc_id))

        if calc.with_embargo or not calc.published:
            if g.user is None:
                abort(401, message='Not logged in to access %s/%s.' % (upload_id, calc_id))

            is_owner = g.user.user_id == 0
            if not is_owner:
                for owner in calc.owners:
                    if owner.user_id == str(g.user.user_id):
                        is_owner = True
                        break
            if not is_owner:
                abort(401, message='Not authorized to access %s/%s.' % (upload_id, calc_id))

        return calc.to_dict(), 200
Markus Scheidgen's avatar
Markus Scheidgen committed
67
68
69
70


repo_calcs_model = api.model('RepoCalculations', {
    'pagination': fields.Nested(pagination_model),
71
72
73
    'results': fields.List(fields.Raw, description=(
        'A list of search results. Each result is a dict with quantitie names as key and '
        'values as values')),
74
    'scroll_id': fields.String(description='Id of the current scroll view in scroll based search.'),
75
    'aggregations': fields.Raw(description=(
Markus Scheidgen's avatar
Markus Scheidgen committed
76
        'A dict with all aggregations. Each aggregation is dictionary with a metrics dict as '
77
78
        'value and quantity value as key. The metrics are code runs(calcs), %s. ' %
        ', '.join(search.metrics_names))),
Markus Scheidgen's avatar
Markus Scheidgen committed
79
    'metrics': fields.Raw(description=(
80
81
        'A dict with the overall metrics. The metrics are code runs(calcs), %s.' %
        ', '.join(search.metrics_names)))
Markus Scheidgen's avatar
Markus Scheidgen committed
82
83
84
85
86
87
})

repo_request_parser = pagination_request_parser.copy()
repo_request_parser.add_argument(
    'owner', type=str,
    help='Specify which calcs to return: ``all``, ``user``, ``staging``, default is ``all``')
88
89
90
91
repo_request_parser.add_argument(
    'scroll', type=bool, help='Enable scrolling')
repo_request_parser.add_argument(
    'scroll_id', type=str, help='The id of the current scrolling window to use.')
Markus Scheidgen's avatar
Markus Scheidgen committed
92
93
94
repo_request_parser.add_argument(
    'total_metrics', type=str, help=(
        'Metrics to aggregate all search results over.'
95
        'Possible values are %s.' % ', '.join(search.metrics_names)))
Markus Scheidgen's avatar
Markus Scheidgen committed
96
97
98
repo_request_parser.add_argument(
    'aggregation_metrics', type=str, help=(
        'Metrics to aggregate all aggregation buckets over as comma separated list. '
99
        'Possible values are %s.' % ', '.join(search.metrics_names)))
Markus Scheidgen's avatar
Markus Scheidgen committed
100

101
102
103
104
for search_quantity in search.search_quantities.keys():
    _, _, description = search.search_quantities[search_quantity]
    repo_request_parser.add_argument(search_quantity, type=str, help=description)

Markus Scheidgen's avatar
Markus Scheidgen committed
105
106
107

@ns.route('/')
class RepoCalcsResource(Resource):
108
    @api.doc('search')
109
    @api.response(400, 'Invalid requests, e.g. wrong owner type or bad quantities')
Markus Scheidgen's avatar
Markus Scheidgen committed
110
111
112
113
114
    @api.expect(repo_request_parser, validate=True)
    @api.marshal_with(repo_calcs_model, skip_none=True, code=200, description='Metadata send')
    @login_if_available
    def get(self):
        """
115
116
117
118
119
120
121
122
        Search for calculations in the repository from, paginated.

        The ``owner`` parameter determines the overall entries to search through.
        You can use the various quantities to search/filter for. For some of the
        indexed quantities this endpoint returns aggregation information. This means
        you will be given a list of all possible values and the number of entries
        that have the certain value. You can also use these aggregations on an empty
        search to determine the possible values.
123
124
125
126
127
128
129
130
131
132
133
134
135
136

        The pagination parameters allows determine which page to return via the
        ``page`` and ``per_page`` parameters. Pagination however, is limited to the first
        100k (depending on ES configuration) hits. An alternative to pagination is to use
        ``scroll`` and ``scroll_id``. With ``scroll`` you will get a ``scroll_id`` on
        the first request. Each call with ``scroll`` and the respective ``scroll_id`` will
        return the next ``per_page`` (here the default is 1000) results. Scroll however,
        ignores ordering and does not return aggregations. The scroll view used in the
        background will stay alive for 1 minute between requests.

        The search will return aggregations on a predefined set of quantities. Aggregations
        will tell you what quantity values exist and how many entries match those values.

        Ordering is determined by ``order_by`` and ``order`` parameters.
137
        """
138
139

        try:
140
141
            scroll = bool(request.args.get('scroll', False))
            scroll_id = request.args.get('scroll_id', None)
142
            page = int(request.args.get('page', 1))
143
            per_page = int(request.args.get('per_page', 10 if not scroll else 1000))
144
            order = int(request.args.get('order', -1))
Markus Scheidgen's avatar
Markus Scheidgen committed
145
146
147
148
149
            total_metrics_str = request.args.get('total_metrics', '')
            aggregation_metrics_str = request.args.get('aggregation_metrics', '')

            total_metrics = [
                metric for metric in total_metrics_str.split(',')
150
                if metric in search.metrics_names]
Markus Scheidgen's avatar
Markus Scheidgen committed
151
152
            aggregation_metrics = [
                metric for metric in aggregation_metrics_str.split(',')
153
                if metric in search.metrics_names]
154
155
156
        except Exception:
            abort(400, message='bad parameter types')

157
        owner = request.args.get('owner', 'all')
158
        order_by = request.args.get('order_by', 'formula')
159
160

        try:
161
            assert page >= 1
162
163
164
165
            assert per_page > 0
        except AssertionError:
            abort(400, message='invalid pagination')

166
167
168
        if order not in [-1, 1]:
            abort(400, message='invalid pagination')

169
        if owner == 'all':
170
171
172
            q = Q('term', published=True) & Q('term', with_embargo=False)
            if g.user is not None:
                q = q | Q('term', owners__user_id=g.user.user_id)
173
174
175
176
        elif owner == 'user':
            if g.user is None:
                abort(401, message='Authentication required for owner value user.')

177
            q = Q('term', owners__user_id=g.user.user_id)
178
179
180
        elif owner == 'staging':
            if g.user is None:
                abort(401, message='Authentication required for owner value user.')
181
            q = Q('term', published=False) & Q('term', owners__user_id=g.user.user_id)
182
183
184
        else:
            abort(400, message='Invalid owner value. Valid values are all|user|staging, default is all')

185
186
        data = dict(**request.args)
        data.pop('owner', None)
187
188
189
190
191
192
        data.pop('scroll', None)
        data.pop('scroll_id', None)
        data.pop('per_page', None)
        data.pop('page', None)
        data.pop('order', None)
        data.pop('order_by', None)
Markus Scheidgen's avatar
Markus Scheidgen committed
193
194
        data.pop('total_metrics', None)
        data.pop('aggregation_metrics', None)
195
196
197
198

        if scroll:
            data.update(scroll_id=scroll_id, size=per_page)
        else:
Markus Scheidgen's avatar
Markus Scheidgen committed
199
200
201
            data.update(
                per_page=per_page, page=page, order=order, order_by=order_by,
                total_metrics=total_metrics, aggregation_metrics=aggregation_metrics)
202
203

        try:
204
205
206
            if scroll:
                page = -1
                scroll_id, total, results = search.scroll_search(q=q, **data)
Markus Scheidgen's avatar
Markus Scheidgen committed
207
208
                aggregations = {}
                metrics = {}
209
210
            else:
                scroll_id = None
Markus Scheidgen's avatar
Markus Scheidgen committed
211
                total, results, aggregations, metrics = search.aggregate_search(q=q, **data)
212
213
214
215
216
217
        except KeyError as e:
            abort(400, str(e))

        return dict(
            pagination=dict(total=total, page=page, per_page=per_page),
            results=results,
218
            scroll_id=scroll_id,
Markus Scheidgen's avatar
Markus Scheidgen committed
219
220
            aggregations=aggregations,
            metrics=metrics), 200