Commit a9105a2b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch '760-elasticsearch-upgrade' into 'v1.0.6'

Resolve "ElasticSearch upgrade for v1.x"

See merge request !584
parents 33312917 2c4b04cd
Pipeline #126852 passed with stages
in 45 minutes and 52 seconds
......@@ -83,7 +83,7 @@ python tests:
services:
- name: rabbitmq:3.9.13
alias: rabbitmq
- name: docker.elastic.co/elasticsearch/elasticsearch:6.8.15
- name: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
alias: elastic
# fix issue with running elastic in gitlab ci runner:
# https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
......
......@@ -100,7 +100,7 @@ services:
# the search engine
elastic:
restart: always
image: docker.elastic.co/elasticsearch/elasticsearch:6.8.15
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
container_name: nomad_oasis_elastic
environment:
- discovery.type=single-node
......
This diff is collapsed.
......@@ -449,8 +449,6 @@ class Index():
self.index_config_key = index_config_key
def __elasticsearch_operation(self, name: str, *args, **kwargs):
if 'doc_type' not in kwargs:
kwargs['doc_type'] = self.doc_type.name
if 'index' not in kwargs:
kwargs['index'] = self.index_name
......@@ -493,15 +491,12 @@ class Index():
}
}
},
'mappings': {
self.doc_type.name: self.doc_type.mapping
}
'mappings': self.doc_type.mapping
})
logger.info('elasticsearch index created')
elif upsert:
self.elastic_client.indices.put_mapping(
index=self.index_name,
doc_type=self.doc_type.name,
body=self.doc_type.mapping)
logger.info('elasticsearch index updated')
else:
......
......@@ -34,7 +34,7 @@ partially implemented.
from typing import Union, List, Iterable, Any, cast, Dict, Iterator, Generator, Callable
import json
import elasticsearch
import elasticsearch.helpers
from elasticsearch.exceptions import TransportError, RequestError
from elasticsearch_dsl import Q, A, Search
from elasticsearch_dsl.query import Query as EsQuery
......@@ -226,13 +226,13 @@ def update_metadata(
yield dict(
doc=entry_doc,
_id=entry_metadata.entry_id,
_type=entry_index.doc_type.name,
_index=entry_index.index_name,
_op_type='update')
updates = list(elastic_updates())
_, failed = elasticsearch.helpers.bulk(
infrastructure.elastic_client, updates, stats_only=True)
failed = cast(int, failed)
if update_materials:
# TODO update the matrials index at least for v1
......@@ -894,9 +894,9 @@ def _es_to_api_aggregation(
entries = None
if 'entries' in es_bucket:
if longest_nested_key:
entries = [{longest_nested_key: item['_source']} for item in es_bucket.entries.hits.hits]
entries = [{longest_nested_key: item['_source'].to_dict()} for item in es_bucket.entries.hits.hits]
else:
entries = [item['_source'] for item in es_bucket.entries.hits.hits]
entries = [item['_source'].to_dict() for item in es_bucket.entries.hits.hits]
# By default ES returns values of 0 and 1 for terms aggregation
# targeting boolean values. Here we transform them into True/False
......@@ -1047,7 +1047,7 @@ def search(
if order_field != doc_type.id_field:
sort[doc_type.id_field] = pagination.order.value
search = search.sort(sort)
search = search.extra(size=pagination.page_size)
search = search.extra(size=pagination.page_size, track_total_hits=True)
if pagination.page_offset:
search = search.extra(**{'from': pagination.page_offset})
......@@ -1124,7 +1124,7 @@ def search(
# pagination
next_page_after_value = None
if 0 < len(es_response.hits) < es_response.hits.total and len(es_response.hits) >= pagination.page_size:
if 0 < len(es_response.hits) < es_response.hits.total.value and len(es_response.hits) >= pagination.page_size:
last = es_response.hits[-1]
if order_field == doc_type.id_field:
next_page_after_value = last[doc_type.id_field]
......@@ -1135,7 +1135,7 @@ def search(
after_value = last.meta.sort[0]
next_page_after_value = '%s:%s' % (after_value, last[doc_type.id_field])
pagination_response = PaginationResponse(
total=es_response.hits.total,
total=es_response.hits.total.value,
next_page_after_value=next_page_after_value,
**pagination.dict())
......
......@@ -45,7 +45,7 @@ services:
# the search engine
elastic:
restart: 'no'
image: docker.elastic.co/elasticsearch/elasticsearch:6.8.15
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
container_name: nomad_elastic
environment:
- cluster.routing.allocation.disk.threshold_enabled=true
......
......@@ -36,7 +36,7 @@ services:
# the search engine
elastic:
restart: always
image: docker.elastic.co/elasticsearch/elasticsearch:6.8.15
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
container_name: nomad_oasis_elastic
environment:
- discovery.type=single-node
......
......@@ -16,7 +16,7 @@ worker:
routing: "queue"
elastic:
host: elasticsearch.elasticsearch.svc.cluster.local
host: elasticsearch.elasticsearch-7.svc.cluster.local
mongo:
host: rs0/mongodb-0.mongo.mongodb.svc.cluster.local,mongodb-1.mongo.mongodb.svc.cluster.local,mongodb-2.mongo.mongodb.svc.cluster.local
......
......@@ -36,7 +36,7 @@ worker:
nomadNodeType: "prod-worker"
elastic:
host: elasticsearch.elasticsearch.svc.cluster.local
host: elasticsearch.elasticsearch-7.svc.cluster.local
port: 9200
mongo:
......
......@@ -28,7 +28,7 @@ worker:
nomadNodeType: "prod-worker"
elastic:
host: elasticsearch.elasticsearch.svc.cluster.local
host: elasticsearch.elasticsearch-7.svc.cluster.local
port: 9200
mongo:
......
......@@ -36,7 +36,7 @@ worker:
nomadNodeType: "prod-worker"
elastic:
host: elasticsearch.elasticsearch.svc.cluster.local
host: elasticsearch.elasticsearch-7.svc.cluster.local
port: 9200
mongo:
......
......@@ -19,7 +19,7 @@ pytz==2021.1
aniso8601==7.0.0
ase==3.19.0
python-keycloak==0.26.1
elasticsearch-dsl==6.4.0
elasticsearch-dsl==7.4.0
pydantic==1.8.2
jmespath==0.10.0
httpx==0.22.0
......@@ -47,7 +47,7 @@ bitarray==2.3.5
# [infrastructure]
optimade[mongo]==0.14.0
structlog==20.1.0
elasticsearch==6.4.0
elasticsearch==7.17.1
msgpack==0.5.6
celery[redis]==4.4.7
mongoengine==0.19.1
......@@ -111,4 +111,4 @@ devtools==0.8.0
mkdocs==1.2.3
mkdocs-material==8.1.1
mkdocs-material-extensions==1.0.3
mkdocs-macros-plugin==0.6.3
\ No newline at end of file
mkdocs-macros-plugin==0.6.3
......@@ -190,7 +190,7 @@ class TestEditRepo():
assert self.mongo(1, reviewers=[])
self.assert_elastic(1, comment=None)
self.assert_elastic(1, references=None)
self.assert_elastic(1, references=[])
self.assert_elastic(1, authors=[test_user.user_id])
self.assert_elastic(1, viewers=[test_user.user_id])
......
......@@ -52,9 +52,9 @@ def assert_search_upload(
index=config.elastic.entries_index, body=body)['hits']
if size != -1:
assert search_results['total'] == size
assert search_results['total']['value'] == size
if search_results['total'] > 0:
if search_results['total']['value'] > 0:
for hit in search_results['hits']:
hit = utils.flat(hit['_source'])
for key, value in kwargs.items():
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment