Added mirror api endpoint to query for uploads. Adopted cli command respectively.

nomad/api/mirror.py

@@ -16,9 +16,10 @@
 The mirror API of the nomad@FAIRDI APIs. Allows to export upload metadata.
 """
 
+from flask import request
 from flask_restplus import Resource, abort, fields
 
-from nomad import processing
+from nomad import processing as proc
 
 from .app import api
 from .auth import admin_login_required
@@ -34,6 +35,35 @@ mirror_upload_model = api.model('MirrorUpload', {
     'upload_files_path': fields.String(description='The path to the local uploads file folder')
 })
 
+mirror_query_model = api.model('MirrorQuery', {
+    'query': fields.Raw(
+        description='Mongoengine query that is used to search for uploads to mirror.')
+})
+
+
+@ns.route('/')
+class MirrorUploadsResource(Resource):
+    @api.doc('get_uploads_mirror')
+    @api.response(400, 'Bad query')
+    @api.marshal_with(
+        mirror_upload_model, skip_none=True, code=200, as_list=True,
+        description='Uploads exported')
+    @api.expect(mirror_query_model)
+    @admin_login_required
+    def post(self):
+        json_data = request.get_json()
+        if json_data is None:
+            query = {}
+        else:
+            query = json_data.get('query', {})
+
+        try:
+            return [
+                dict(upload_id=upload.upload_id)
+                for upload in proc.Upload.objects(**query)], 200
+        except Exception as e:
+            abort(400, message='Could not query mongodb: %s' % str(e))
+
 
 @upload_route(ns)
 class MirrorUploadResource(Resource):
@@ -47,7 +77,7 @@ class MirrorUploadResource(Resource):
         Export upload (and all calc) metadata for mirrors.
         """
         try:
-            upload = processing.Upload.get(upload_id)
+            upload = proc.Upload.get(upload_id)
         except KeyError:
             abort(404, message='Upload with id %s does not exist.' % upload_id)
 

nomad/cli/client/mirror.py

@@ -98,72 +98,66 @@ def mirror(query, move: bool, dry: bool, source_mapping, target_mapping):
     from nomad.cli.client import create_client
     client = create_client()
 
-    while True:
-        query_results = client.repo.quantity_search(quantity='upload_id', **query).response().result
-        upload_ids = query_results.quantities['upload_id']
-
-        for upload_id in upload_ids['values'].keys():
-            upload_data = client.mirror.get_upload_mirror(upload_id=upload_id).response().result
-
-            try:
-                upload = proc.Upload.get(upload_id)
-                if __in_test:
-                    proc.Calc.objects(upload_id=upload_id).delete()
-                    proc.Upload.objects(upload_id=upload_id).delete()
-                    search.delete_upload(upload_id)
-
-                    raise KeyError()
-
-                print(
-                    'Upload %s already exists, updating existing uploads is not implemented yet. '
-                    'Skip upload.' % upload_id)
-                continue
-            except KeyError:
-                pass
-
-            if dry:
-                print(
-                    'Need to mirror %s with %d calcs at %s' %
-                    (upload_data.upload_id, upload_ids['values'][upload_id], upload_data.upload_files_path))
-                continue
-
-            # copy/mv file
-            upload_files_path = upload_data.upload_files_path
+    uploads = client.mirror.get_uploads_mirror(payload=dict(query={})).response().result
+
+    for upload_with_id in uploads:
+        upload_id = upload_with_id['upload_id']
+        upload_data = client.mirror.get_upload_mirror(upload_id=upload_id).response().result
+
+        try:
+            upload = proc.Upload.get(upload_id)
             if __in_test:
-                tmp = os.path.join(config.fs.tmp, 'to_mirror')
-                os.rename(upload_files_path, tmp)
-                upload_files_path = tmp
-
-            upload_files_path = source_mapping.apply(upload_files_path)
-
-            target_upload_files_path = files.PathObject(
-                config.fs.public, upload_id, create_prefix=True, prefix=True).os_path
-            target_upload_files_path = target_mapping.apply(target_upload_files_path)
-            if not os.path.exists(target_upload_files_path):
-                os.makedirs(target_upload_files_path)
-
-            if move:
-                os.rename(upload_files_path, target_upload_files_path)
-                os.symlink(os.path.abspath(target_upload_files_path), upload_files_path)
-            else:
-                for to_copy in os.listdir(upload_files_path):
-                    shutil.copyfile(
-                        os.path.join(upload_files_path, to_copy),
-                        os.path.join(target_upload_files_path, to_copy))
-
-            # create mongo
-            upload = proc.Upload.from_json(upload_data.upload, created=True).save()
-            for calc_data in upload_data.calcs:
-                proc.Calc.from_json(calc_data, created=True).save()
-
-            # index es
-            search.index_all(upload.to_upload_with_metadata().calcs)
+                proc.Calc.objects(upload_id=upload_id).delete()
+                proc.Upload.objects(upload_id=upload_id).delete()
+                search.delete_upload(upload_id)
 
-            print(
-                'Mirrored %s with %d calcs at %s' %
-                (upload_data.upload_id, upload_ids['values'][upload_id], upload_data.upload_files_path))
+                raise KeyError()
 
-        if 'after' not in upload_ids:
-            break
+            print(
+                'Upload %s already exists, updating existing uploads is not implemented yet. '
+                'Skip upload.' % upload_id)
+            continue
+        except KeyError:
+            pass
 
-        query.update(after=upload_ids['after'])
+        if dry:
+            print(
+                'Need to mirror %s with %d calcs at %s' %
+                (upload_data.upload_id, len(upload_data.calcs), upload_data.upload_files_path))
+            continue
+
+        # copy/mv file
+        upload_files_path = upload_data.upload_files_path
+        if __in_test:
+            tmp = os.path.join(config.fs.tmp, 'to_mirror')
+            os.rename(upload_files_path, tmp)
+            upload_files_path = tmp
+
+        upload_files_path = source_mapping.apply(upload_files_path)
+
+        target_upload_files_path = files.PathObject(
+            config.fs.public, upload_id, create_prefix=True, prefix=True).os_path
+        target_upload_files_path = target_mapping.apply(target_upload_files_path)
+        if not os.path.exists(target_upload_files_path):
+            os.makedirs(target_upload_files_path)
+
+        if move:
+            os.rename(upload_files_path, target_upload_files_path)
+            os.symlink(os.path.abspath(target_upload_files_path), upload_files_path)
+        else:
+            for to_copy in os.listdir(upload_files_path):
+                shutil.copyfile(
+                    os.path.join(upload_files_path, to_copy),
+                    os.path.join(target_upload_files_path, to_copy))
+
+        # create mongo
+        upload = proc.Upload.from_json(upload_data.upload, created=True).save()
+        for calc_data in upload_data.calcs:
+            proc.Calc.from_json(calc_data, created=True).save()
+
+        # index es
+        search.index_all(upload.to_upload_with_metadata().calcs)
+
+        print(
+            'Mirrored %s with %d calcs at %s' %
+            (upload_data.upload_id, len(upload_data.calcs), upload_data.upload_files_path))

tests/test_api.py

@@ -1103,6 +1103,15 @@ class TestMirror:
         assert len(data['calcs']) == len(published.calcs)
         assert data['upload_files_path'] == published.upload_files.os_path
 
+    def test_uploads(self, client, published, admin_user_auth, no_warn):
+        rv = client.post(
+            '/mirror/',
+            content_type='application/json', data='{"query":{}}', headers=admin_user_auth)
+        assert rv.status_code == 200, rv.data
+
+        data = json.loads(rv.data)
+        assert data[0]['upload_id'] == published.upload_id
+
 
 def test_docs(client):
     rv = client.get('/docs/index.html')