Commit 2c4b04cd authored by Lauri Himanen's avatar Lauri Himanen Committed by Markus Scheidgen
Browse files

Resolve "ElasticSearch upgrade for v1.x"

parent 718cadf5
...@@ -83,7 +83,7 @@ python tests: ...@@ -83,7 +83,7 @@ python tests:
services: services:
- name: rabbitmq:3.9.13 - name: rabbitmq:3.9.13
alias: rabbitmq alias: rabbitmq
- name: docker.elastic.co/elasticsearch/elasticsearch:6.8.15 - name: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
alias: elastic alias: elastic
# fix issue with running elastic in gitlab ci runner: # fix issue with running elastic in gitlab ci runner:
# https://gitlab.com/gitlab-org/gitlab-ce/issues/42214 # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
......
...@@ -100,7 +100,7 @@ services: ...@@ -100,7 +100,7 @@ services:
# the search engine # the search engine
elastic: elastic:
restart: always restart: always
image: docker.elastic.co/elasticsearch/elasticsearch:6.8.15 image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
container_name: nomad_oasis_elastic container_name: nomad_oasis_elastic
environment: environment:
- discovery.type=single-node - discovery.type=single-node
......
This diff is collapsed.
...@@ -449,8 +449,6 @@ class Index(): ...@@ -449,8 +449,6 @@ class Index():
self.index_config_key = index_config_key self.index_config_key = index_config_key
def __elasticsearch_operation(self, name: str, *args, **kwargs): def __elasticsearch_operation(self, name: str, *args, **kwargs):
if 'doc_type' not in kwargs:
kwargs['doc_type'] = self.doc_type.name
if 'index' not in kwargs: if 'index' not in kwargs:
kwargs['index'] = self.index_name kwargs['index'] = self.index_name
...@@ -493,15 +491,12 @@ class Index(): ...@@ -493,15 +491,12 @@ class Index():
} }
} }
}, },
'mappings': { 'mappings': self.doc_type.mapping
self.doc_type.name: self.doc_type.mapping
}
}) })
logger.info('elasticsearch index created') logger.info('elasticsearch index created')
elif upsert: elif upsert:
self.elastic_client.indices.put_mapping( self.elastic_client.indices.put_mapping(
index=self.index_name, index=self.index_name,
doc_type=self.doc_type.name,
body=self.doc_type.mapping) body=self.doc_type.mapping)
logger.info('elasticsearch index updated') logger.info('elasticsearch index updated')
else: else:
......
...@@ -34,7 +34,7 @@ partially implemented. ...@@ -34,7 +34,7 @@ partially implemented.
from typing import Union, List, Iterable, Any, cast, Dict, Iterator, Generator, Callable from typing import Union, List, Iterable, Any, cast, Dict, Iterator, Generator, Callable
import json import json
import elasticsearch import elasticsearch.helpers
from elasticsearch.exceptions import TransportError, RequestError from elasticsearch.exceptions import TransportError, RequestError
from elasticsearch_dsl import Q, A, Search from elasticsearch_dsl import Q, A, Search
from elasticsearch_dsl.query import Query as EsQuery from elasticsearch_dsl.query import Query as EsQuery
...@@ -226,13 +226,13 @@ def update_metadata( ...@@ -226,13 +226,13 @@ def update_metadata(
yield dict( yield dict(
doc=entry_doc, doc=entry_doc,
_id=entry_metadata.entry_id, _id=entry_metadata.entry_id,
_type=entry_index.doc_type.name,
_index=entry_index.index_name, _index=entry_index.index_name,
_op_type='update') _op_type='update')
updates = list(elastic_updates()) updates = list(elastic_updates())
_, failed = elasticsearch.helpers.bulk( _, failed = elasticsearch.helpers.bulk(
infrastructure.elastic_client, updates, stats_only=True) infrastructure.elastic_client, updates, stats_only=True)
failed = cast(int, failed)
if update_materials: if update_materials:
# TODO update the matrials index at least for v1 # TODO update the matrials index at least for v1
...@@ -894,9 +894,9 @@ def _es_to_api_aggregation( ...@@ -894,9 +894,9 @@ def _es_to_api_aggregation(
entries = None entries = None
if 'entries' in es_bucket: if 'entries' in es_bucket:
if longest_nested_key: if longest_nested_key:
entries = [{longest_nested_key: item['_source']} for item in es_bucket.entries.hits.hits] entries = [{longest_nested_key: item['_source'].to_dict()} for item in es_bucket.entries.hits.hits]
else: else:
entries = [item['_source'] for item in es_bucket.entries.hits.hits] entries = [item['_source'].to_dict() for item in es_bucket.entries.hits.hits]
# By default ES returns values of 0 and 1 for terms aggregation # By default ES returns values of 0 and 1 for terms aggregation
# targeting boolean values. Here we transform them into True/False # targeting boolean values. Here we transform them into True/False
...@@ -1047,7 +1047,7 @@ def search( ...@@ -1047,7 +1047,7 @@ def search(
if order_field != doc_type.id_field: if order_field != doc_type.id_field:
sort[doc_type.id_field] = pagination.order.value sort[doc_type.id_field] = pagination.order.value
search = search.sort(sort) search = search.sort(sort)
search = search.extra(size=pagination.page_size) search = search.extra(size=pagination.page_size, track_total_hits=True)
if pagination.page_offset: if pagination.page_offset:
search = search.extra(**{'from': pagination.page_offset}) search = search.extra(**{'from': pagination.page_offset})
...@@ -1124,7 +1124,7 @@ def search( ...@@ -1124,7 +1124,7 @@ def search(
# pagination # pagination
next_page_after_value = None next_page_after_value = None
if 0 < len(es_response.hits) < es_response.hits.total and len(es_response.hits) >= pagination.page_size: if 0 < len(es_response.hits) < es_response.hits.total.value and len(es_response.hits) >= pagination.page_size:
last = es_response.hits[-1] last = es_response.hits[-1]
if order_field == doc_type.id_field: if order_field == doc_type.id_field:
next_page_after_value = last[doc_type.id_field] next_page_after_value = last[doc_type.id_field]
...@@ -1135,7 +1135,7 @@ def search( ...@@ -1135,7 +1135,7 @@ def search(
after_value = last.meta.sort[0] after_value = last.meta.sort[0]
next_page_after_value = '%s:%s' % (after_value, last[doc_type.id_field]) next_page_after_value = '%s:%s' % (after_value, last[doc_type.id_field])
pagination_response = PaginationResponse( pagination_response = PaginationResponse(
total=es_response.hits.total, total=es_response.hits.total.value,
next_page_after_value=next_page_after_value, next_page_after_value=next_page_after_value,
**pagination.dict()) **pagination.dict())
......
...@@ -45,7 +45,7 @@ services: ...@@ -45,7 +45,7 @@ services:
# the search engine # the search engine
elastic: elastic:
restart: 'no' restart: 'no'
image: docker.elastic.co/elasticsearch/elasticsearch:6.8.15 image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
container_name: nomad_elastic container_name: nomad_elastic
environment: environment:
- cluster.routing.allocation.disk.threshold_enabled=true - cluster.routing.allocation.disk.threshold_enabled=true
......
...@@ -36,7 +36,7 @@ services: ...@@ -36,7 +36,7 @@ services:
# the search engine # the search engine
elastic: elastic:
restart: always restart: always
image: docker.elastic.co/elasticsearch/elasticsearch:6.8.15 image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
container_name: nomad_oasis_elastic container_name: nomad_oasis_elastic
environment: environment:
- discovery.type=single-node - discovery.type=single-node
......
...@@ -16,7 +16,7 @@ worker: ...@@ -16,7 +16,7 @@ worker:
routing: "queue" routing: "queue"
elastic: elastic:
host: elasticsearch.elasticsearch.svc.cluster.local host: elasticsearch.elasticsearch-7.svc.cluster.local
mongo: mongo:
host: rs0/mongodb-0.mongo.mongodb.svc.cluster.local,mongodb-1.mongo.mongodb.svc.cluster.local,mongodb-2.mongo.mongodb.svc.cluster.local host: rs0/mongodb-0.mongo.mongodb.svc.cluster.local,mongodb-1.mongo.mongodb.svc.cluster.local,mongodb-2.mongo.mongodb.svc.cluster.local
......
...@@ -36,7 +36,7 @@ worker: ...@@ -36,7 +36,7 @@ worker:
nomadNodeType: "prod-worker" nomadNodeType: "prod-worker"
elastic: elastic:
host: elasticsearch.elasticsearch.svc.cluster.local host: elasticsearch.elasticsearch-7.svc.cluster.local
port: 9200 port: 9200
mongo: mongo:
......
...@@ -28,7 +28,7 @@ worker: ...@@ -28,7 +28,7 @@ worker:
nomadNodeType: "prod-worker" nomadNodeType: "prod-worker"
elastic: elastic:
host: elasticsearch.elasticsearch.svc.cluster.local host: elasticsearch.elasticsearch-7.svc.cluster.local
port: 9200 port: 9200
mongo: mongo:
......
...@@ -36,7 +36,7 @@ worker: ...@@ -36,7 +36,7 @@ worker:
nomadNodeType: "prod-worker" nomadNodeType: "prod-worker"
elastic: elastic:
host: elasticsearch.elasticsearch.svc.cluster.local host: elasticsearch.elasticsearch-7.svc.cluster.local
port: 9200 port: 9200
mongo: mongo:
......
...@@ -19,7 +19,7 @@ pytz==2021.1 ...@@ -19,7 +19,7 @@ pytz==2021.1
aniso8601==7.0.0 aniso8601==7.0.0
ase==3.19.0 ase==3.19.0
python-keycloak==0.26.1 python-keycloak==0.26.1
elasticsearch-dsl==6.4.0 elasticsearch-dsl==7.4.0
pydantic==1.8.2 pydantic==1.8.2
jmespath==0.10.0 jmespath==0.10.0
httpx==0.22.0 httpx==0.22.0
...@@ -47,7 +47,7 @@ bitarray==2.3.5 ...@@ -47,7 +47,7 @@ bitarray==2.3.5
# [infrastructure] # [infrastructure]
optimade[mongo]==0.14.0 optimade[mongo]==0.14.0
structlog==20.1.0 structlog==20.1.0
elasticsearch==6.4.0 elasticsearch==7.17.1
msgpack==0.5.6 msgpack==0.5.6
celery[redis]==4.4.7 celery[redis]==4.4.7
mongoengine==0.19.1 mongoengine==0.19.1
...@@ -111,4 +111,4 @@ devtools==0.8.0 ...@@ -111,4 +111,4 @@ devtools==0.8.0
mkdocs==1.2.3 mkdocs==1.2.3
mkdocs-material==8.1.1 mkdocs-material==8.1.1
mkdocs-material-extensions==1.0.3 mkdocs-material-extensions==1.0.3
mkdocs-macros-plugin==0.6.3 mkdocs-macros-plugin==0.6.3
\ No newline at end of file
...@@ -190,7 +190,7 @@ class TestEditRepo(): ...@@ -190,7 +190,7 @@ class TestEditRepo():
assert self.mongo(1, reviewers=[]) assert self.mongo(1, reviewers=[])
self.assert_elastic(1, comment=None) self.assert_elastic(1, comment=None)
self.assert_elastic(1, references=None) self.assert_elastic(1, references=[])
self.assert_elastic(1, authors=[test_user.user_id]) self.assert_elastic(1, authors=[test_user.user_id])
self.assert_elastic(1, viewers=[test_user.user_id]) self.assert_elastic(1, viewers=[test_user.user_id])
......
...@@ -52,9 +52,9 @@ def assert_search_upload( ...@@ -52,9 +52,9 @@ def assert_search_upload(
index=config.elastic.entries_index, body=body)['hits'] index=config.elastic.entries_index, body=body)['hits']
if size != -1: if size != -1:
assert search_results['total'] == size assert search_results['total']['value'] == size
if search_results['total'] > 0: if search_results['total']['value'] > 0:
for hit in search_results['hits']: for hit in search_results['hits']:
hit = utils.flat(hit['_source']) hit = utils.flat(hit['_source'])
for key, value in kwargs.items(): for key, value in kwargs.items():
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment