Implemented building of python code snippets in archive and repo

nomad/app/api/archive.py

@@ -33,7 +33,7 @@ from nomad import utils, search
 from .auth import authenticate, create_authorization_predicate
 from .api import api
 from .repo import search_request_parser, add_query
-from .common import calc_route, streamed_zipfile
+from .common import calc_route, streamed_zipfile, build_snippet, to_json
 
 ns = api.namespace(
     'archive',
@@ -112,6 +112,10 @@ archives_from_query_parser = search_request_parser.copy()
 archives_from_query_parser.add_argument(
     name='compress', type=bool, help='Use compression on .zip files, default is not.',
     location='args')
+archives_from_query_parser.add_argument(
+    name='res_type', type=str, help='Type of return value, can be zip of json.',
+    location='args'
+)
 
 
 @ns.route('/query')
@@ -138,6 +142,7 @@ class ArchiveQueryResource(Resource):
         try:
             args = archives_from_query_parser.parse_args()
             compress = args.get('compress', False)
+            res_type = args.get('res_type', 'zip')
         except Exception:
             abort(400, message='bad parameter types')
 
@@ -192,8 +197,16 @@ class ArchiveQueryResource(Resource):
                 lambda *args: BytesIO(manifest_contents),
                 lambda *args: len(manifest_contents))
 
-        return streamed_zipfile(
-            generator(), zipfile_name='nomad_archive.zip', compress=compress)
+        if res_type == 'zip':
+            return streamed_zipfile(
+                generator(), zipfile_name='nomad_archive.zip', compress=compress)
+        elif res_type == 'json':
+            archive_data = to_json(generator())
+            code_snippet = build_snippet(args, os.path.join(api.base_url, ns.name, 'query'))
+            data = {'archive_data': archive_data, 'code_snippet': code_snippet}
+            return data, 200
+        else:
+            raise Exception('Unknown res_type %s' % res_type)
 
 
 @ns.route('/metainfo/<string:metainfo_package_name>')

nomad/app/api/common.py

@@ -16,10 +16,11 @@
 Common data, variables, decorators, models used throughout the API.
 """
 from typing import Callable, IO, Set, Tuple, Iterable
-from flask_restplus import fields
+from flask_restplus import fields, abort
 import zipstream
 from flask import stream_with_context, Response
 import sys
+import json
 
 from nomad.app.utils import RFC3339DateTime
 from nomad.files import Restricted
@@ -153,3 +154,42 @@ def streamed_zipfile(
     response = Response(stream_with_context(generator()), mimetype='application/zip')
     response.headers['Content-Disposition'] = 'attachment; filename={}'.format(zipfile_name)
     return response
+
+
+def to_json(files: Iterable[Tuple[str, str, Callable[[str], IO], Callable[[str], int]]]):
+    data = {}
+    for _, file_id, open_io, _ in files:
+        try:
+            f = open_io(file_id)
+            data[file_id] = json.loads(f.read())
+        except KeyError:
+            pass
+        except Restricted:
+            abort(401, message='Not authorized to access %s.' % file_id)
+    return data
+
+
+def build_snippet(args, base_url):
+    str_code = 'import requests\n'
+    str_code += 'from urllib.parse import urlencode\n'
+    str_code += '\n\n'
+    str_code += 'def query_repository(args, base_url):\n'
+    str_code += '    url = "%s?%s" % (base_url, urlencode(args))\n'
+    str_code += '    response = requests.get(url)\n'
+    str_code += '    if response.status_code != 200:\n'
+    str_code += '        raise Exception("nomad return status %d" % response.status_code)\n'
+    str_code += '    return response.json()\n'
+    str_code += '\n\n'
+    str_code += 'args = {'
+    for key, val in args.items():
+        if val is None:
+            continue
+        if isinstance(val, str):
+            str_code += '"%s": "%s", ' % (key, val)
+        else:
+            str_code += '"%s": %s, ' % (key, val)
+    str_code += '}\n'
+    str_code += 'base_url = "%s"\n' % base_url
+    str_code += 'JSON_DATA = query_repository(args, base_url)\n'
+
+    return str_code

nomad/app/api/repo.py

@@ -24,6 +24,7 @@ from elasticsearch_dsl import Q
 from elasticsearch.exceptions import NotFoundError
 import elasticsearch.helpers
 from datetime import datetime
+import os.path
 
 from nomad import search, utils, datamodel, processing as proc, infrastructure
 from nomad.app.utils import rfc3339DateTime, RFC3339DateTime, with_logger
@@ -32,7 +33,7 @@ from nomad.datamodel import UserMetadata, Dataset, User
 
 from .api import api
 from .auth import authenticate
-from .common import pagination_model, pagination_request_parser, calc_route
+from .common import pagination_model, pagination_request_parser, calc_route, build_snippet
 
 ns = api.namespace('repo', description='Access repository metadata.')
 
@@ -80,6 +81,8 @@ repo_calcs_model_fields = {
         'value and quantity value as key. The possible metrics are code runs(calcs), %s. '
         'There is a pseudo quantity "total" with a single value "all" that contains the '
         ' metrics over all results. ' % ', '.join(datamodel.Domain.instance.metrics_names))),
+    'code_snippet': fields.String(description=(
+        'A string of python code snippet which can be executed to reproduce the api result.')),
 }
 for group_name, (group_quantity, _) in search.groups.items():
     repo_calcs_model_fields[group_name] = fields.Nested(api.model('RepoDatasets', {
@@ -300,6 +303,10 @@ class RepoCalcsResource(Resource):
                     if args.get(group_name, False):
                         results[group_name] = quantities[group_quantity]
 
+            # build python code snippet
+            snippet = build_snippet(args, os.path.join(api.base_url, ns.name, ''))
+            results['code_snippet'] = snippet
+
             return results, 200
         except search.ScrollIdNotFound:
             abort(400, 'The given scroll_id does not exist.')

tests/app/test_api.py

@@ -667,6 +667,16 @@ class TestArchive(UploadFilesBasedTests):
         assert rv.status_code == 200
         assert_zip_file(rv, files=1)
 
+    def test_code_snippet(self, api, processeds, test_user_auth):
+        query_params = {'atoms': 'Si', 'res_type': 'json'}
+        url = '/archive/query?%s' % urlencode(query_params)
+        rv = api.get(url, headers=test_user_auth)
+
+        assert rv.status_code == 200
+        data = json.loads(rv.data)
+        assert isinstance(data, dict)
+        assert data['code_snippet'] is not None
+
 
 class TestRepo():
     @pytest.fixture(scope='class')
@@ -1058,6 +1068,14 @@ class TestRepo():
         data = json.loads(rv.data)
         assert data['pagination']['total'] > 0
 
+    def test_code_snippet(self, api, example_elastic_calcs, test_user_auth):
+        rv = api.get('/repo/?per_page=10', headers=test_user_auth)
+        assert rv.status_code == 200
+        data = json.loads(rv.data)
+        assert data['code_snippet'] is not None
+        # exec does not seem to work
+        # exec(data['code_snippet'])
+
 
 class TestEditRepo():