Commit 7fce1f3a authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'scrolling-patch' into 'master'

added small patch for scrolling feature of elasticsearch

See merge request !5
parents 5648a421 0aa55c9b
......@@ -24,6 +24,8 @@ from onelogin.saml2.utils import OneLogin_Saml2_Utils
from redis import Redis
from urllib.parse import urlparse
import re
from elasticsearch import TransportError
from app import es_config
from . import api
......@@ -185,6 +187,11 @@ def get_materials(page, per_page):
if not json_data:
raise InvalidEncRequest("No input data provided", 400)
# check for ecxeeded elasticsearch pagination limit
scroll = json_data.get('search_by').get('pagination') == 'scroll'
if page*per_page >= 1e4 and not scroll:
raise InvalidEncRequest('Too many results requested. Limit is 10000, or use "pagination":"scoll" and subsequently scroll_id.', 510)
# create the match part
match_query = {}
search_by = json_data.get('search_by')
......@@ -304,11 +311,29 @@ def get_materials(page, per_page):
'from': (page - 1) * per_page,
'query': es_query
}
if scroll:
scroll_id = search_by.get('scroll_id')
es_request_body = {
'size': per_page,
'query': es_query
}
if scroll_id is None:
es_search_results = es.search(index = es_config['index'], doc_type = 'material', scroll = '1m', body = es_request_body)
else:
es_request_body['scroll_id'] = str(scroll_id)
try:
es_search_results = es.scroll(scroll_id = scroll_id, scroll = '1m')
except TransportError as e:
raise InvalidEncRequest("Scrolling error: %s" % str(e), 510)
materials_found_list = [render_es_result(hit['_source']) for hit in es_search_results['hits']['hits']]
return materials_found_list, -1, { 'es_query': es_query, 'scroll_id': scroll_id }
scroll_id = es_search_results['_scroll_id']
else:
es_search_results = es.search(index=es_config['index'], doc_type='material', body=es_request_body)
total_es_search_results = es_search_results['hits']['total']
materials_found_list = [render_es_result(hit['_source']) for hit in es_search_results['hits']['hits']]
if scroll:
return materials_found_list, total_es_search_results, { 'es_query': es_query, 'scroll_id': scroll_id }
return materials_found_list, total_es_search_results, { 'es_query': es_query }
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment