Merge branch '760-elasticsearch-upgrade' into 'v1.0.6'

Resolve "ElasticSearch upgrade for v1.x"

See merge request !584

.gitlab-ci.yml

@@ -83,7 +83,7 @@ python tests:
   services:
     - name: rabbitmq:3.9.13
       alias: rabbitmq
-    - name: docker.elastic.co/elasticsearch/elasticsearch:6.8.15
+    - name: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
       alias: elastic
       # fix issue with running elastic in gitlab ci runner:
       # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214

docs/oasis.md

@@ -100,7 +100,7 @@ services:
     # the search engine
     elastic:
         restart: always
-        image: docker.elastic.co/elasticsearch/elasticsearch:6.8.15
+        image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
         container_name: nomad_oasis_elastic
         environment:
             - discovery.type=single-node

nomad/metainfo/elasticsearch_extension.py

@@ -449,8 +449,6 @@ class Index():
         self.index_config_key = index_config_key
 
     def __elasticsearch_operation(self, name: str, *args, **kwargs):
-        if 'doc_type' not in kwargs:
-            kwargs['doc_type'] = self.doc_type.name
         if 'index' not in kwargs:
             kwargs['index'] = self.index_name
 
@@ -493,15 +491,12 @@ class Index():
                         }
                     }
                 },
-                'mappings': {
-                    self.doc_type.name: self.doc_type.mapping
-                }
+                'mappings': self.doc_type.mapping
             })
             logger.info('elasticsearch index created')
         elif upsert:
             self.elastic_client.indices.put_mapping(
                 index=self.index_name,
-                doc_type=self.doc_type.name,
                 body=self.doc_type.mapping)
             logger.info('elasticsearch index updated')
         else:

nomad/search.py

@@ -34,7 +34,7 @@ partially implemented.
 
 from typing import Union, List, Iterable, Any, cast, Dict, Iterator, Generator, Callable
 import json
-import elasticsearch
+import elasticsearch.helpers
 from elasticsearch.exceptions import TransportError, RequestError
 from elasticsearch_dsl import Q, A, Search
 from elasticsearch_dsl.query import Query as EsQuery
@@ -226,13 +226,13 @@ def update_metadata(
             yield dict(
                 doc=entry_doc,
                 _id=entry_metadata.entry_id,
-                _type=entry_index.doc_type.name,
                 _index=entry_index.index_name,
                 _op_type='update')
 
     updates = list(elastic_updates())
     _, failed = elasticsearch.helpers.bulk(
         infrastructure.elastic_client, updates, stats_only=True)
+    failed = cast(int, failed)
 
     if update_materials:
         # TODO update the matrials index at least for v1
@@ -894,9 +894,9 @@ def _es_to_api_aggregation(
             entries = None
             if 'entries' in es_bucket:
                 if longest_nested_key:
-                    entries = [{longest_nested_key: item['_source']} for item in es_bucket.entries.hits.hits]
+                    entries = [{longest_nested_key: item['_source'].to_dict()} for item in es_bucket.entries.hits.hits]
                 else:
-                    entries = [item['_source'] for item in es_bucket.entries.hits.hits]
+                    entries = [item['_source'].to_dict() for item in es_bucket.entries.hits.hits]
 
             # By default ES returns values of 0 and 1 for terms aggregation
             # targeting boolean values. Here we transform them into True/False
@@ -1047,7 +1047,7 @@ def search(
     if order_field != doc_type.id_field:
         sort[doc_type.id_field] = pagination.order.value
     search = search.sort(sort)
-    search = search.extra(size=pagination.page_size)
+    search = search.extra(size=pagination.page_size, track_total_hits=True)
 
     if pagination.page_offset:
         search = search.extra(**{'from': pagination.page_offset})
@@ -1124,7 +1124,7 @@ def search(
 
     # pagination
     next_page_after_value = None
-    if 0 < len(es_response.hits) < es_response.hits.total and len(es_response.hits) >= pagination.page_size:
+    if 0 < len(es_response.hits) < es_response.hits.total.value and len(es_response.hits) >= pagination.page_size:
         last = es_response.hits[-1]
         if order_field == doc_type.id_field:
             next_page_after_value = last[doc_type.id_field]
@@ -1135,7 +1135,7 @@ def search(
             after_value = last.meta.sort[0]
             next_page_after_value = '%s:%s' % (after_value, last[doc_type.id_field])
     pagination_response = PaginationResponse(
-        total=es_response.hits.total,
+        total=es_response.hits.total.value,
         next_page_after_value=next_page_after_value,
         **pagination.dict())
 

ops/docker-compose/infrastructure/docker-compose.yml

@@ -45,7 +45,7 @@ services:
     # the search engine
     elastic:
         restart: 'no'
-        image: docker.elastic.co/elasticsearch/elasticsearch:6.8.15
+        image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
         container_name: nomad_elastic
         environment:
             - cluster.routing.allocation.disk.threshold_enabled=true

ops/docker-compose/nomad-oasis/docker-compose.yml

@@ -36,7 +36,7 @@ services:
     # the search engine
     elastic:
         restart: always
-        image: docker.elastic.co/elasticsearch/elasticsearch:6.8.15
+        image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
         container_name: nomad_oasis_elastic
         environment:
             - discovery.type=single-node

ops/helm/nomad/deployments/dev-values.yaml

@@ -16,7 +16,7 @@ worker:
   routing: "queue"
 
 elastic:
-  host: elasticsearch.elasticsearch.svc.cluster.local
+  host: elasticsearch.elasticsearch-7.svc.cluster.local
 
 mongo:
   host: rs0/mongodb-0.mongo.mongodb.svc.cluster.local,mongodb-1.mongo.mongodb.svc.cluster.local,mongodb-2.mongo.mongodb.svc.cluster.local

ops/helm/nomad/deployments/prod-staging-values.yaml

@@ -36,7 +36,7 @@ worker:
   nomadNodeType: "prod-worker"
 
 elastic:
-  host: elasticsearch.elasticsearch.svc.cluster.local
+  host: elasticsearch.elasticsearch-7.svc.cluster.local
   port: 9200
 
 mongo:

ops/helm/nomad/deployments/prod-test-values.yaml

@@ -28,7 +28,7 @@ worker:
   nomadNodeType: "prod-worker"
 
 elastic:
-  host: elasticsearch.elasticsearch.svc.cluster.local
+  host: elasticsearch.elasticsearch-7.svc.cluster.local
   port: 9200
 
 mongo:

ops/helm/nomad/deployments/prod-v1-values.yaml

@@ -36,7 +36,7 @@ worker:
   nomadNodeType: "prod-worker"
 
 elastic:
-  host: elasticsearch.elasticsearch.svc.cluster.local
+  host: elasticsearch.elasticsearch-7.svc.cluster.local
   port: 9200
 
 mongo:

requirements.txt

@@ -19,7 +19,7 @@ pytz==2021.1
 aniso8601==7.0.0
 ase==3.19.0
 python-keycloak==0.26.1
-elasticsearch-dsl==6.4.0
+elasticsearch-dsl==7.4.0
 pydantic==1.8.2
 jmespath==0.10.0
 httpx==0.22.0
@@ -47,7 +47,7 @@ bitarray==2.3.5
 # [infrastructure]
 optimade[mongo]==0.14.0
 structlog==20.1.0
-elasticsearch==6.4.0
+elasticsearch==7.17.1
 msgpack==0.5.6
 celery[redis]==4.4.7
 mongoengine==0.19.1
@@ -111,4 +111,4 @@ devtools==0.8.0
 mkdocs==1.2.3
 mkdocs-material==8.1.1
 mkdocs-material-extensions==1.0.3
-mkdocs-macros-plugin==0.6.3
+mkdocs-macros-plugin==0.6.3
\ No newline at end of file

tests/app/v1/routers/test_entries_edit.py

@@ -190,7 +190,7 @@ class TestEditRepo():
         assert self.mongo(1, reviewers=[])
 
         self.assert_elastic(1, comment=None)
-        self.assert_elastic(1, references=None)
+        self.assert_elastic(1, references=[])
         self.assert_elastic(1, authors=[test_user.user_id])
         self.assert_elastic(1, viewers=[test_user.user_id])
 

tests/test_search.py

@@ -52,9 +52,9 @@ def assert_search_upload(
         index=config.elastic.entries_index, body=body)['hits']
 
     if size != -1:
-        assert search_results['total'] == size
+        assert search_results['total']['value'] == size
 
-    if search_results['total'] > 0:
+    if search_results['total']['value'] > 0:
         for hit in search_results['hits']:
             hit = utils.flat(hit['_source'])
             for key, value in kwargs.items():