Added an uploads cli admin command to re-index. Related to #167. [skip ci]

nomad/cli/admin/uploads.py

@@ -104,6 +104,25 @@ def chown(ctx, user, uploads):
         search.publish(calcs)
 
 
+@uploads.command(help='(Re-)index all calcs of the given uploads.')
+@click.argument('UPLOADS', nargs=-1)
+@click.pass_context
+def index(ctx, uploads):
+    infrastructure.setup_repository_db()
+    _, uploads = query_uploads(ctx, uploads)
+    uploads_count = uploads.count()
+
+    print('%d uploads selected, indexing ...' % uploads_count)
+
+    i, failed = 0, 0
+    for upload in uploads:
+        upload_with_metadata = upload.to_upload_with_metadata()
+        calcs = upload_with_metadata.calcs
+        failed += search.index_all(calcs)
+
+        print('   indexed %d of %d uploads, failed to index %d entries' % (i, uploads_count, failed))
+
+
 @uploads.command(help='Delete selected upload')
 @click.argument('UPLOADS', nargs=-1)
 @click.option('--with-coe-repo', help='Also attempt to delete from repository db', is_flag=True)

nomad/search.py

@@ -178,7 +178,12 @@ def publish(calcs: Iterable[datamodel.CalcWithMetadata]) -> None:
 
 
 def index_all(calcs: Iterable[datamodel.CalcWithMetadata]) -> None:
-    """ Adds all given calcs with their metadata to the index. """
+    """
+    Adds all given calcs with their metadata to the index.
+
+    Returns:
+        Number of failed entries.
+    """
     def elastic_updates():
         for calc in calcs:
             entry = Entry.from_calc_with_metadata(calc)
@@ -186,8 +191,9 @@ def index_all(calcs: Iterable[datamodel.CalcWithMetadata]) -> None:
             entry['_op_type'] = 'index'
             yield entry
 
-    elasticsearch.helpers.bulk(infrastructure.elastic_client, elastic_updates())
+    _, failed = elasticsearch.helpers.bulk(infrastructure.elastic_client, elastic_updates(), stats_only=True)
     refresh()
+    return failed
 
 
 def refresh():

tests/test_cli.py

@@ -66,6 +66,21 @@ class TestAdminUploads:
         assert Upload.objects(upload_id=upload_id).first() is None
         assert Calc.objects(upload_id=upload_id).first() is None
 
+    def test_index(self, published):
+        upload_id = published.upload_id
+        calc = Calc.objects(upload_id=upload_id).first()
+        calc.metadata['comment'] = 'specific'
+        calc.save()
+
+        assert search.entry_search(search_parameters=dict(comment='specific'))['pagination']['total'] == 0
+
+        result = click.testing.CliRunner().invoke(
+            cli, ['admin', 'uploads', 'index', upload_id], catch_exceptions=False, obj=utils.POPO())
+        assert result.exit_code == 0
+        assert 'index' in result.stdout
+
+        assert search.entry_search(search_parameters=dict(comment='specific'))['pagination']['total'] == 1
+
     def test_re_process(self, published, monkeypatch):
         monkeypatch.setattr('nomad.config.version', 'test_version')
         upload_id = published.upload_id