Refactored pagination models

nomad/app/v1/models.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 #
 
+import re
 from typing import List, Dict, Optional, Union, Any, Mapping
 import enum
 from fastapi import Body, Request, HTTPException, Query as FastApiQuery
@@ -441,31 +442,173 @@ class Pagination(BaseModel):
     size: Optional[int] = Field(
         10, description=strip('''
             The page size, e.g. the maximum number of items contained in one response.
-            A `size` of 0 will omit any results.
+            A `size` of 0 will return no results.
         '''))
     order_by: Optional[str] = Field(
-        calc_id,  # type: ignore
+        None,  # type: ignore
         description=strip('''
-            The results are ordered by the values of this field. The response
-            either contains the first `size` value or the next `size` values after `after`.
+            The results are ordered by the values of this field. If omitted, default
+            ordering is applied.
         '''))
     order: Optional[Direction] = Field(
         Direction.asc, description=strip('''
-            The order direction of the results based on `order_by`. Its either
-            ascending `asc` or decending `desc`.
+            The ordering direction of the results based on `order_by`. Its either
+            ascending `asc` or decending `desc`. Default is `asc`.
         '''))
     after: Optional[str] = Field(
         None, description=strip('''
-            A request for the page after this value, i.e. the next `size` values behind `after`.
-            This depends on the `order_by`.
-            Each response contains the `after` value for the *next* request following
-            the defined order.
-
-            The after value and its type depends on the API operation and potentially on
-            the `order_by` field and its type.
-            The after value will always be a string encoded value. It might be an `order_by` value, or an index.
-            The after value might contain an id as *tie breaker*, if `order_by` is not the unique.
+            A string value which defines a position in the total list of results. If a
+            value for `after` is provided when making a request, the response will return
+            the next `size` results, starting from the point specified by `after`. If
+            the value is omitted when making a request, the response will just give the
+            first page of results.
+
+            The response should also contain an attribute `next_after`, which similarly
+            defines the starting position of the next page. Thus, one would normally start
+            with a request where `after` is omitted, then use the `next_after` value from
+            the responses as the `after` in the next request, to get the next page of
+            results.
+
+            Note that the values of `after` and `next_after` depends on the API operation
+            and potentially on the `order_by` field and its type.
+            It will always be a string encoded value. It might be an `order_by` value, or an index.
+            It might contain an id as *tie breaker*, if `order_by` is not a field with
+            unique values.
             The *tie breaker* will be `:` separated, e.g. `<value>:<id>`.
+            For simple, index-based pagination, àfter` will be the zero-based index of the
+            first result in the response.
+        '''))
+    page: Optional[int] = Field(
+        None, description=strip('''
+            For simple, index-based pagination, this should contain the number of the
+            requested page (1-based). When provided in a request, this attribute can be
+            used instead of `after` to jump to a particular results page. However, if you
+            specify both `after` *and* `page` in your request, they need to be consistent.
+        '''))
+
+    @validator('size')
+    def validate_size(cls, size):  # pylint: disable=no-self-argument
+        assert size >= 0, 'size must be >= 0'
+        return size
+
+    @validator('order_by')
+    def validate_order_by(cls, order_by):  # pylint: disable=no-self-argument
+        '''
+        Override this in your Pagination class to ensure that a valid attribute is selected.
+        This method has to be implemented!
+        '''
+        raise NotImplementedError('Validation of `order_by` not implemented!')
+
+    @validator('after')
+    def validate_after(cls, after, values):  # pylint: disable=no-self-argument
+        '''
+        Override this in your Pagination class to implement validation of the `after` value.
+        This method has to be implemented!
+        '''
+        raise NotImplementedError('Validation of `after` not implemented!')
+
+    @validator('page')
+    def validate_page(cls, page, values):  # pylint: disable=no-self-argument
+        if page is not None:
+            # This attribute is not expected unless we are using an IndexBasedPagination
+            # or in the PaginationResponse of an index-based paginated request.
+            raise AssertionError('Value for `page` not permitted')
+        return page
+
+
+class IndexBasedPagination(Pagination):
+    @validator('after')
+    def validate_after(cls, after, values):  # pylint: disable=no-self-argument
+        # This is validated in the root validator instead
+        return after
+
+    @validator('page')
+    def validate_page(cls, page, values):  # pylint: disable=no-self-argument
+        # This is validated in the root validator instead
+        return page
+
+    @root_validator(skip_on_failure=True)
+    def validate_values(cls, values):  # pylint: disable=no-self-argument
+        '''
+        Ensure that both page and after are filled in consistently. This requires us to
+        look at `page`, `after` and `size` (whichever is set). If inconsistent
+        information is provided, an exception will be thrown.
+        '''
+        page = values.get('page')
+        after = values.get('after')
+        size = values.get('size')
+        if after is not None:
+            try:
+                after = int(after)
+            except ValueError:
+                raise ValueError('Invalid value for `after` - could not convert to integer.')
+        if page is None and after is None:
+            # Neither page nor after provided - default to first page
+            page = 1
+            after = 0
+        elif page is not None and after is not None:
+            # Both provided - check that they are consistent.
+            assert after == (page - 1) * size, 'inconsistent page/after values provided'
+        elif page is not None:
+            # Only page provided - calculate after
+            after = (page - 1) * size
+        elif after is not None:
+            # Only after provided - calculate page
+            if not size:
+                assert after == 0, 'after must be zero if size is zero.'
+                page = 1
+            else:
+                assert after % size == 0, 'after must be a multiple of size'
+                page = after // size + 1
+        assert page >= 1, 'negative paging is not allowed'
+        values['page'] = page
+        values['after'] = str(after)
+        return values
+
+
+class PaginationResponse(Pagination):
+    total: int = Field(
+        ..., description=strip('''
+        The total number of results that fit the given query. This is independent of
+        any pagination and aggregations.
+        '''))
+    next_after: Optional[str] = Field(
+        None, description=strip('''
+        The *next* value to be used as `after` in a follow up requests, to get the next
+        page of results. If no more results are available, `next_after` will not be set.
+        '''))
+    next_url: Optional[str] = Field(
+        None, description=strip('''
+        The url to get the next page.
+        '''))
+
+    @validator('order_by')
+    def validate_order_by(cls, order_by):  # pylint: disable=no-self-argument
+        # No validation - behaviour of this field depends on api method
+        return order_by
+
+    @validator('after')
+    def validate_after(cls, after, values):  # pylint: disable=no-self-argument
+        # No validation - behaviour of this field depends on api method
+        return after
+
+    @validator('page')
+    def validate_page(cls, page, values):  # pylint: disable=no-self-argument
+        # No validation - behaviour of this field depends on api method
+        return page
+
+    @validator('next_after')
+    def validate_next_after(cls, next_after, values):  # pylint: disable=no-self-argument
+        # No validation - behaviour of this field depends on api method
+        return next_after
+
+
+class EntryPagination(Pagination):
+    order_by: Optional[str] = Field(
+        calc_id,  # type: ignore
+        description=strip('''
+            The results are ordered by the values of this field. If omitted, default
+            ordering is applied.
         '''))
 
     @validator('order_by')
@@ -478,11 +621,6 @@ class Pagination(BaseModel):
         assert quantity.definition.is_scalar, 'the order_by quantity must be a scalar'
         return order_by
 
-    @validator('size')
-    def validate_size(cls, size):  # pylint: disable=no-self-argument
-        assert size >= 0, 'size must be positive integer'
-        return size
-
     @validator('after')
     def validate_after(cls, after, values):  # pylint: disable=no-self-argument
         order_by = values.get('order_by', calc_id)
@@ -491,18 +629,33 @@ class Pagination(BaseModel):
         return after
 
 
-pagination_parameters = parameter_dependency_from_model(
-    'pagination_parameters', Pagination)
+entry_pagination_parameters = parameter_dependency_from_model(
+    'entry_pagination_parameters', EntryPagination)
 
 
-class AggregationPagination(Pagination):
+class AggregationPagination(EntryPagination):
     order_by: Optional[str] = Field(
-        None, description=strip('''
-        The search results are ordered by the values of this quantity. The response
-        either contains the first `size` value or the next `size` values after `after`.
+        None,  # type: ignore
+        description=strip('''
+            The results are ordered by the values of this field. If omitted, default
+            ordering is applied.
         '''))
 
 
+class DatasetPagination(IndexBasedPagination):
+    @validator('order_by')
+    def validate_order_by(cls, order_by):  # pylint: disable=no-self-argument
+        # TODO: need real validation
+        if order_by is None:
+            return order_by
+        assert re.match('^[a-zA-Z0-9_]+$', order_by), 'order_by must be alphanumeric'
+        return order_by
+
+
+dataset_pagination_parameters = parameter_dependency_from_model(
+    'dataset_pagination_parameters', DatasetPagination)
+
+
 class AggregatedEntities(BaseModel):
     size: Optional[pydantic.conint(gt=0)] = Field(  # type: ignore
         1, description=strip('''
@@ -590,7 +743,7 @@ class Statistic(BaseModel):
 
 
 class WithQueryAndPagination(WithQuery):
-    pagination: Optional[Pagination] = Body(
+    pagination: Optional[EntryPagination] = Body(
         None,
         example={
             'size': 5,
@@ -773,7 +926,7 @@ class EntriesArchiveDownload(WithQuery):
 
 
 class EntriesRaw(WithQuery):
-    pagination: Optional[Pagination] = Body(None)
+    pagination: Optional[EntryPagination] = Body(None)
 
 
 class EntriesRawDownload(WithQuery):
@@ -784,17 +937,6 @@ class EntriesRawDownload(WithQuery):
         })
 
 
-class PaginationResponse(Pagination):
-    total: int = Field(..., description=strip('''
-        The total number of entries that fit the given `query`. This is independent of
-        any pagination and aggregations.
-    '''))
-    next_after: Optional[str] = Field(None, description=strip('''
-        The *next* after value to be used as `after` in a follow up requests for the
-        next page of results.
-    '''))
-
-
 class StatisticResponse(Statistic):
     data: Dict[str, Dict[str, int]] = Field(
         None, description=strip('''
@@ -828,7 +970,7 @@ class CodeResponse(BaseModel):
 
 
 class EntriesMetadataResponse(EntriesMetadata):
-    pagination: PaginationResponse
+    pagination: PaginationResponse  # type: ignore
     statistics: Optional[Dict[str, StatisticResponse]]  # type: ignore
     aggregations: Optional[Dict[str, AggregationResponse]]  # type: ignore
     data: List[Dict[str, Any]] = Field(
@@ -851,7 +993,7 @@ class EntryRaw(BaseModel):
 
 
 class EntriesRawResponse(EntriesRaw):
-    pagination: PaginationResponse = Field(None)
+    pagination: PaginationResponse = Field(None)  # type: ignore
     data: List[EntryRaw] = Field(None)
 
 
@@ -875,7 +1017,7 @@ class EntryArchive(BaseModel):
 
 
 class EntriesArchiveResponse(EntriesArchive):
-    pagination: PaginationResponse = Field(None)
+    pagination: PaginationResponse = Field(None)  # type: ignore
     data: List[EntryArchive] = Field(None)
 
 

nomad/app/v1/routers/datasets.py

@@ -32,7 +32,7 @@ from .auth import get_required_user
 from .entries import _do_exaustive_search
 from ..utils import create_responses
 from ..models import (
-    pagination_parameters, Pagination, PaginationResponse, Query, HTTPExceptionModel,
+    dataset_pagination_parameters, DatasetPagination, PaginationResponse, Query, HTTPExceptionModel,
     User, Direction, Owner, Any_)
 
 
@@ -110,7 +110,7 @@ async def get_datasets(
         name: str = FastApiQuery(None),
         user_id: str = FastApiQuery(None),
         dataset_type: str = FastApiQuery(None),
-        pagination: Pagination = Depends(pagination_parameters)):
+        pagination: DatasetPagination = Depends(dataset_pagination_parameters)):
     '''
     Retrieves all datasets that match the given criteria.
     '''
@@ -125,14 +125,12 @@ async def get_datasets(
 
     mongodb_query = mongodb_query.order_by(order_by)
 
-    start = 0
-    if pagination.after is not None:
-        start = int(pagination.after)
+    start = int(pagination.after)
     end = start + pagination.size
 
     pagination_response = PaginationResponse(
         total=mongodb_query.count(),
-        next_after=str(end),
+        next_after=str(end) if end < mongodb_query.count() else None,
         **pagination.dict())  # type: ignore
 
     return {

nomad/app/v1/routers/entries.py

@@ -33,9 +33,9 @@ from nomad.archive import (
 from .auth import get_optional_user
 from ..utils import create_streamed_zipfile, File, create_responses
 from ..models import (
-    Pagination, WithQuery, MetadataRequired, EntriesMetadataResponse, EntriesMetadata,
+    EntryPagination, WithQuery, MetadataRequired, EntriesMetadataResponse, EntriesMetadata,
     EntryMetadataResponse, query_parameters, metadata_required_parameters, Files, Query,
-    pagination_parameters, files_parameters, User, Owner, HTTPExceptionModel, EntriesRaw,
+    entry_pagination_parameters, files_parameters, User, Owner, HTTPExceptionModel, EntriesRaw,
     EntriesRawResponse, EntriesRawDownload, EntryRaw, EntryRawFile, EntryRawResponse,
     EntriesArchiveDownload, EntryArchiveResponse, EntriesArchive, EntriesArchiveResponse,
     ArchiveRequired)
@@ -140,7 +140,7 @@ async def post_entries_metadata_query(
     response_model_exclude_none=True)
 async def get_entries_metadata(
         with_query: WithQuery = Depends(query_parameters),
-        pagination: Pagination = Depends(pagination_parameters),
+        pagination: EntryPagination = Depends(entry_pagination_parameters),
         required: MetadataRequired = Depends(metadata_required_parameters),
         user: User = Depends(get_optional_user)):
     '''
@@ -166,7 +166,7 @@ def _do_exaustive_search(owner: Owner, query: Query, include: List[str], user: U
     while True:
         response = perform_search(
             owner=owner, query=query,
-            pagination=Pagination(size=100, after=after, order_by='upload_id'),
+            pagination=EntryPagination(size=100, after=after, order_by='upload_id'),
             required=MetadataRequired(include=include),
             user_id=user.user_id if user is not None else None)
 
@@ -218,7 +218,7 @@ def _create_entry_raw(entry_metadata: Dict[str, Any], uploads: _Uploads):
 
 
 def _answer_entries_raw_request(
-        owner: Owner, query: Query, pagination: Pagination, user: User):
+        owner: Owner, query: Query, pagination: EntryPagination, user: User):
 
     if owner == Owner.all_:
         raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=strip('''
@@ -256,7 +256,7 @@ def _answer_entries_raw_download_request(owner: Owner, query: Query, files: File
 
     response = perform_search(
         owner=owner, query=query,
-        pagination=Pagination(size=0),
+        pagination=EntryPagination(size=0),
         required=MetadataRequired(include=[]),
         user_id=user.user_id if user is not None else None)
 
@@ -366,7 +366,7 @@ async def post_entries_raw_query(data: EntriesRaw, user: User = Depends(get_opti
     responses=create_responses(_bad_owner_response))
 async def get_entries_raw(
         with_query: WithQuery = Depends(query_parameters),
-        pagination: Pagination = Depends(pagination_parameters),
+        pagination: EntryPagination = Depends(entry_pagination_parameters),
         user: User = Depends(get_optional_user)):
 
     return _answer_entries_raw_request(
@@ -440,7 +440,7 @@ def _read_archive(entry_metadata, uploads, required):
 
 
 def _answer_entries_archive_request(
-        owner: Owner, query: Query, pagination: Pagination, required: ArchiveRequired,
+        owner: Owner, query: Query, pagination: EntryPagination, required: ArchiveRequired,
         user: User):
 
     if owner == Owner.all_:
@@ -545,7 +545,7 @@ async def post_entries_archive_query(
     responses=create_responses(_bad_owner_response, _bad_archive_required_response))
 async def get_entries_archive_query(
         with_query: WithQuery = Depends(query_parameters),
-        pagination: Pagination = Depends(pagination_parameters),
+        pagination: EntryPagination = Depends(entry_pagination_parameters),
         user: User = Depends(get_optional_user)):
 
     return _answer_entries_archive_request(
@@ -566,7 +566,7 @@ def _answer_entries_archive_download_request(
 
     response = perform_search(
         owner=owner, query=query,
-        pagination=Pagination(size=0),
+        pagination=EntryPagination(size=0),
         required=MetadataRequired(include=[]),
         user_id=user.user_id if user is not None else None)
 

nomad/search.py

@@ -33,7 +33,7 @@ from nomad.metainfo.search_extension import (  # pylint: disable=unused-import
     search_quantities, metrics, order_default_quantities, groups)
 from nomad.app.v1 import models as api_models
 from nomad.app.v1.models import (
-    Pagination, PaginationResponse, Query, MetadataRequired, SearchResponse, Aggregation,
+    EntryPagination, PaginationResponse, Query, MetadataRequired, SearchResponse, Aggregation,
     Statistic, StatisticResponse, AggregationOrderType, AggregationResponse, AggregationDataItem)
 
 
@@ -1067,7 +1067,7 @@ def _es_to_api_aggregation(es_response, name: str, agg: Aggregation) -> Aggregat
 def search(
         owner: str = 'public',
         query: Query = None,
-        pagination: Pagination = None,
+        pagination: EntryPagination = None,
         required: MetadataRequired = None,
         aggregations: Dict[str, Aggregation] = {},
         statistics: Dict[str, Statistic] = {},
@@ -1084,7 +1084,7 @@ def search(
 
     # pagination
     if pagination is None:
-        pagination = Pagination()
+        pagination = EntryPagination()
 
     search = Search(index=config.elastic.index_name)