Commit d1a9e81b authored by David Sikter's avatar David Sikter
Browse files

Refactored pagination models

parent e1594a58
Pipeline #96976 passed with stages
in 23 minutes and 34 seconds
......@@ -16,6 +16,7 @@
# limitations under the License.
#
import re
from typing import List, Dict, Optional, Union, Any, Mapping
import enum
from fastapi import Body, Request, HTTPException, Query as FastApiQuery
......@@ -441,31 +442,173 @@ class Pagination(BaseModel):
size: Optional[int] = Field(
10, description=strip('''
The page size, e.g. the maximum number of items contained in one response.
A `size` of 0 will omit any results.
A `size` of 0 will return no results.
'''))
order_by: Optional[str] = Field(
calc_id, # type: ignore
None, # type: ignore
description=strip('''
The results are ordered by the values of this field. The response
either contains the first `size` value or the next `size` values after `after`.
The results are ordered by the values of this field. If omitted, default
ordering is applied.
'''))
order: Optional[Direction] = Field(
Direction.asc, description=strip('''
The order direction of the results based on `order_by`. Its either
ascending `asc` or decending `desc`.
The ordering direction of the results based on `order_by`. Its either
ascending `asc` or decending `desc`. Default is `asc`.
'''))
after: Optional[str] = Field(
None, description=strip('''
A request for the page after this value, i.e. the next `size` values behind `after`.
This depends on the `order_by`.
Each response contains the `after` value for the *next* request following
the defined order.
The after value and its type depends on the API operation and potentially on
the `order_by` field and its type.
The after value will always be a string encoded value. It might be an `order_by` value, or an index.
The after value might contain an id as *tie breaker*, if `order_by` is not the unique.
A string value which defines a position in the total list of results. If a
value for `after` is provided when making a request, the response will return
the next `size` results, starting from the point specified by `after`. If
the value is omitted when making a request, the response will just give the
first page of results.
The response should also contain an attribute `next_after`, which similarly
defines the starting position of the next page. Thus, one would normally start
with a request where `after` is omitted, then use the `next_after` value from
the responses as the `after` in the next request, to get the next page of
results.
Note that the values of `after` and `next_after` depends on the API operation
and potentially on the `order_by` field and its type.
It will always be a string encoded value. It might be an `order_by` value, or an index.
It might contain an id as *tie breaker*, if `order_by` is not a field with
unique values.
The *tie breaker* will be `:` separated, e.g. `<value>:<id>`.
For simple, index-based pagination, àfter` will be the zero-based index of the
first result in the response.
'''))
page: Optional[int] = Field(
None, description=strip('''
For simple, index-based pagination, this should contain the number of the
requested page (1-based). When provided in a request, this attribute can be
used instead of `after` to jump to a particular results page. However, if you
specify both `after` *and* `page` in your request, they need to be consistent.
'''))
@validator('size')
def validate_size(cls, size): # pylint: disable=no-self-argument
assert size >= 0, 'size must be >= 0'
return size
@validator('order_by')
def validate_order_by(cls, order_by): # pylint: disable=no-self-argument
'''
Override this in your Pagination class to ensure that a valid attribute is selected.
This method has to be implemented!
'''
raise NotImplementedError('Validation of `order_by` not implemented!')
@validator('after')
def validate_after(cls, after, values): # pylint: disable=no-self-argument
'''
Override this in your Pagination class to implement validation of the `after` value.
This method has to be implemented!
'''
raise NotImplementedError('Validation of `after` not implemented!')
@validator('page')
def validate_page(cls, page, values): # pylint: disable=no-self-argument
if page is not None:
# This attribute is not expected unless we are using an IndexBasedPagination
# or in the PaginationResponse of an index-based paginated request.
raise AssertionError('Value for `page` not permitted')
return page
class IndexBasedPagination(Pagination):
@validator('after')
def validate_after(cls, after, values): # pylint: disable=no-self-argument
# This is validated in the root validator instead
return after
@validator('page')
def validate_page(cls, page, values): # pylint: disable=no-self-argument
# This is validated in the root validator instead
return page
@root_validator(skip_on_failure=True)
def validate_values(cls, values): # pylint: disable=no-self-argument
'''
Ensure that both page and after are filled in consistently. This requires us to
look at `page`, `after` and `size` (whichever is set). If inconsistent
information is provided, an exception will be thrown.
'''
page = values.get('page')
after = values.get('after')
size = values.get('size')
if after is not None:
try:
after = int(after)
except ValueError:
raise ValueError('Invalid value for `after` - could not convert to integer.')
if page is None and after is None:
# Neither page nor after provided - default to first page
page = 1
after = 0
elif page is not None and after is not None:
# Both provided - check that they are consistent.
assert after == (page - 1) * size, 'inconsistent page/after values provided'
elif page is not None:
# Only page provided - calculate after
after = (page - 1) * size
elif after is not None:
# Only after provided - calculate page
if not size:
assert after == 0, 'after must be zero if size is zero.'
page = 1
else:
assert after % size == 0, 'after must be a multiple of size'
page = after // size + 1
assert page >= 1, 'negative paging is not allowed'
values['page'] = page
values['after'] = str(after)
return values
class PaginationResponse(Pagination):
total: int = Field(
..., description=strip('''
The total number of results that fit the given query. This is independent of
any pagination and aggregations.
'''))
next_after: Optional[str] = Field(
None, description=strip('''
The *next* value to be used as `after` in a follow up requests, to get the next
page of results. If no more results are available, `next_after` will not be set.
'''))
next_url: Optional[str] = Field(
None, description=strip('''
The url to get the next page.
'''))
@validator('order_by')
def validate_order_by(cls, order_by): # pylint: disable=no-self-argument
# No validation - behaviour of this field depends on api method
return order_by
@validator('after')
def validate_after(cls, after, values): # pylint: disable=no-self-argument
# No validation - behaviour of this field depends on api method
return after
@validator('page')
def validate_page(cls, page, values): # pylint: disable=no-self-argument
# No validation - behaviour of this field depends on api method
return page
@validator('next_after')
def validate_next_after(cls, next_after, values): # pylint: disable=no-self-argument
# No validation - behaviour of this field depends on api method
return next_after
class EntryPagination(Pagination):
order_by: Optional[str] = Field(
calc_id, # type: ignore
description=strip('''
The results are ordered by the values of this field. If omitted, default
ordering is applied.
'''))
@validator('order_by')
......@@ -478,11 +621,6 @@ class Pagination(BaseModel):
assert quantity.definition.is_scalar, 'the order_by quantity must be a scalar'
return order_by
@validator('size')
def validate_size(cls, size): # pylint: disable=no-self-argument
assert size >= 0, 'size must be positive integer'
return size
@validator('after')
def validate_after(cls, after, values): # pylint: disable=no-self-argument
order_by = values.get('order_by', calc_id)
......@@ -491,18 +629,33 @@ class Pagination(BaseModel):
return after
pagination_parameters = parameter_dependency_from_model(
'pagination_parameters', Pagination)
entry_pagination_parameters = parameter_dependency_from_model(
'entry_pagination_parameters', EntryPagination)
class AggregationPagination(Pagination):
class AggregationPagination(EntryPagination):
order_by: Optional[str] = Field(
None, description=strip('''
The search results are ordered by the values of this quantity. The response
either contains the first `size` value or the next `size` values after `after`.
None, # type: ignore
description=strip('''
The results are ordered by the values of this field. If omitted, default
ordering is applied.
'''))
class DatasetPagination(IndexBasedPagination):
@validator('order_by')
def validate_order_by(cls, order_by): # pylint: disable=no-self-argument
# TODO: need real validation
if order_by is None:
return order_by
assert re.match('^[a-zA-Z0-9_]+$', order_by), 'order_by must be alphanumeric'
return order_by
dataset_pagination_parameters = parameter_dependency_from_model(
'dataset_pagination_parameters', DatasetPagination)
class AggregatedEntities(BaseModel):
size: Optional[pydantic.conint(gt=0)] = Field( # type: ignore
1, description=strip('''
......@@ -590,7 +743,7 @@ class Statistic(BaseModel):
class WithQueryAndPagination(WithQuery):
pagination: Optional[Pagination] = Body(
pagination: Optional[EntryPagination] = Body(
None,
example={
'size': 5,
......@@ -773,7 +926,7 @@ class EntriesArchiveDownload(WithQuery):
class EntriesRaw(WithQuery):
pagination: Optional[Pagination] = Body(None)
pagination: Optional[EntryPagination] = Body(None)
class EntriesRawDownload(WithQuery):
......@@ -784,17 +937,6 @@ class EntriesRawDownload(WithQuery):
})
class PaginationResponse(Pagination):
total: int = Field(..., description=strip('''
The total number of entries that fit the given `query`. This is independent of
any pagination and aggregations.
'''))
next_after: Optional[str] = Field(None, description=strip('''
The *next* after value to be used as `after` in a follow up requests for the
next page of results.
'''))
class StatisticResponse(Statistic):
data: Dict[str, Dict[str, int]] = Field(
None, description=strip('''
......@@ -828,7 +970,7 @@ class CodeResponse(BaseModel):
class EntriesMetadataResponse(EntriesMetadata):
pagination: PaginationResponse
pagination: PaginationResponse # type: ignore
statistics: Optional[Dict[str, StatisticResponse]] # type: ignore
aggregations: Optional[Dict[str, AggregationResponse]] # type: ignore
data: List[Dict[str, Any]] = Field(
......@@ -851,7 +993,7 @@ class EntryRaw(BaseModel):
class EntriesRawResponse(EntriesRaw):
pagination: PaginationResponse = Field(None)
pagination: PaginationResponse = Field(None) # type: ignore
data: List[EntryRaw] = Field(None)
......@@ -875,7 +1017,7 @@ class EntryArchive(BaseModel):
class EntriesArchiveResponse(EntriesArchive):
pagination: PaginationResponse = Field(None)
pagination: PaginationResponse = Field(None) # type: ignore
data: List[EntryArchive] = Field(None)
......
......@@ -32,7 +32,7 @@ from .auth import get_required_user
from .entries import _do_exaustive_search
from ..utils import create_responses
from ..models import (
pagination_parameters, Pagination, PaginationResponse, Query, HTTPExceptionModel,
dataset_pagination_parameters, DatasetPagination, PaginationResponse, Query, HTTPExceptionModel,
User, Direction, Owner, Any_)
......@@ -110,7 +110,7 @@ async def get_datasets(
name: str = FastApiQuery(None),
user_id: str = FastApiQuery(None),
dataset_type: str = FastApiQuery(None),
pagination: Pagination = Depends(pagination_parameters)):
pagination: DatasetPagination = Depends(dataset_pagination_parameters)):
'''
Retrieves all datasets that match the given criteria.
'''
......@@ -125,14 +125,12 @@ async def get_datasets(
mongodb_query = mongodb_query.order_by(order_by)
start = 0
if pagination.after is not None:
start = int(pagination.after)
start = int(pagination.after)
end = start + pagination.size
pagination_response = PaginationResponse(
total=mongodb_query.count(),
next_after=str(end),
next_after=str(end) if end < mongodb_query.count() else None,
**pagination.dict()) # type: ignore
return {
......
......@@ -33,9 +33,9 @@ from nomad.archive import (
from .auth import get_optional_user
from ..utils import create_streamed_zipfile, File, create_responses
from ..models import (
Pagination, WithQuery, MetadataRequired, EntriesMetadataResponse, EntriesMetadata,
EntryPagination, WithQuery, MetadataRequired, EntriesMetadataResponse, EntriesMetadata,
EntryMetadataResponse, query_parameters, metadata_required_parameters, Files, Query,
pagination_parameters, files_parameters, User, Owner, HTTPExceptionModel, EntriesRaw,
entry_pagination_parameters, files_parameters, User, Owner, HTTPExceptionModel, EntriesRaw,
EntriesRawResponse, EntriesRawDownload, EntryRaw, EntryRawFile, EntryRawResponse,
EntriesArchiveDownload, EntryArchiveResponse, EntriesArchive, EntriesArchiveResponse,
ArchiveRequired)
......@@ -140,7 +140,7 @@ async def post_entries_metadata_query(
response_model_exclude_none=True)
async def get_entries_metadata(
with_query: WithQuery = Depends(query_parameters),
pagination: Pagination = Depends(pagination_parameters),
pagination: EntryPagination = Depends(entry_pagination_parameters),
required: MetadataRequired = Depends(metadata_required_parameters),
user: User = Depends(get_optional_user)):
'''
......@@ -166,7 +166,7 @@ def _do_exaustive_search(owner: Owner, query: Query, include: List[str], user: U
while True:
response = perform_search(
owner=owner, query=query,
pagination=Pagination(size=100, after=after, order_by='upload_id'),
pagination=EntryPagination(size=100, after=after, order_by='upload_id'),
required=MetadataRequired(include=include),
user_id=user.user_id if user is not None else None)
......@@ -218,7 +218,7 @@ def _create_entry_raw(entry_metadata: Dict[str, Any], uploads: _Uploads):
def _answer_entries_raw_request(
owner: Owner, query: Query, pagination: Pagination, user: User):
owner: Owner, query: Query, pagination: EntryPagination, user: User):
if owner == Owner.all_:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=strip('''
......@@ -256,7 +256,7 @@ def _answer_entries_raw_download_request(owner: Owner, query: Query, files: File
response = perform_search(
owner=owner, query=query,
pagination=Pagination(size=0),
pagination=EntryPagination(size=0),
required=MetadataRequired(include=[]),
user_id=user.user_id if user is not None else None)
......@@ -366,7 +366,7 @@ async def post_entries_raw_query(data: EntriesRaw, user: User = Depends(get_opti
responses=create_responses(_bad_owner_response))
async def get_entries_raw(
with_query: WithQuery = Depends(query_parameters),
pagination: Pagination = Depends(pagination_parameters),
pagination: EntryPagination = Depends(entry_pagination_parameters),
user: User = Depends(get_optional_user)):
return _answer_entries_raw_request(
......@@ -440,7 +440,7 @@ def _read_archive(entry_metadata, uploads, required):
def _answer_entries_archive_request(
owner: Owner, query: Query, pagination: Pagination, required: ArchiveRequired,
owner: Owner, query: Query, pagination: EntryPagination, required: ArchiveRequired,
user: User):
if owner == Owner.all_:
......@@ -545,7 +545,7 @@ async def post_entries_archive_query(
responses=create_responses(_bad_owner_response, _bad_archive_required_response))
async def get_entries_archive_query(
with_query: WithQuery = Depends(query_parameters),
pagination: Pagination = Depends(pagination_parameters),
pagination: EntryPagination = Depends(entry_pagination_parameters),
user: User = Depends(get_optional_user)):
return _answer_entries_archive_request(
......@@ -566,7 +566,7 @@ def _answer_entries_archive_download_request(
response = perform_search(
owner=owner, query=query,
pagination=Pagination(size=0),
pagination=EntryPagination(size=0),
required=MetadataRequired(include=[]),
user_id=user.user_id if user is not None else None)
......
......@@ -33,7 +33,7 @@ from nomad.metainfo.search_extension import ( # pylint: disable=unused-import
search_quantities, metrics, order_default_quantities, groups)
from nomad.app.v1 import models as api_models
from nomad.app.v1.models import (
Pagination, PaginationResponse, Query, MetadataRequired, SearchResponse, Aggregation,
EntryPagination, PaginationResponse, Query, MetadataRequired, SearchResponse, Aggregation,
Statistic, StatisticResponse, AggregationOrderType, AggregationResponse, AggregationDataItem)
......@@ -1067,7 +1067,7 @@ def _es_to_api_aggregation(es_response, name: str, agg: Aggregation) -> Aggregat
def search(
owner: str = 'public',
query: Query = None,
pagination: Pagination = None,
pagination: EntryPagination = None,
required: MetadataRequired = None,
aggregations: Dict[str, Aggregation] = {},
statistics: Dict[str, Statistic] = {},
......@@ -1084,7 +1084,7 @@ def search(
# pagination
if pagination is None:
pagination = Pagination()
pagination = EntryPagination()
search = Search(index=config.elastic.index_name)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment