Commit e2d16c2e authored by Alvin Noe Ladines's avatar Alvin Noe Ladines
Browse files

Implemented building of python code snippets in archive and repo

parent 3f98dac7
Pipeline #67142 failed with stages
in 14 minutes and 20 seconds
......@@ -33,7 +33,7 @@ from nomad import utils, search
from .auth import authenticate, create_authorization_predicate
from .api import api
from .repo import search_request_parser, add_query
from .common import calc_route, streamed_zipfile
from .common import calc_route, streamed_zipfile, build_snippet, to_json
ns = api.namespace(
'archive',
......@@ -112,6 +112,10 @@ archives_from_query_parser = search_request_parser.copy()
archives_from_query_parser.add_argument(
name='compress', type=bool, help='Use compression on .zip files, default is not.',
location='args')
archives_from_query_parser.add_argument(
name='res_type', type=str, help='Type of return value, can be zip of json.',
location='args'
)
@ns.route('/query')
......@@ -138,6 +142,7 @@ class ArchiveQueryResource(Resource):
try:
args = archives_from_query_parser.parse_args()
compress = args.get('compress', False)
res_type = args.get('res_type', 'zip')
except Exception:
abort(400, message='bad parameter types')
......@@ -192,8 +197,16 @@ class ArchiveQueryResource(Resource):
lambda *args: BytesIO(manifest_contents),
lambda *args: len(manifest_contents))
return streamed_zipfile(
generator(), zipfile_name='nomad_archive.zip', compress=compress)
if res_type == 'zip':
return streamed_zipfile(
generator(), zipfile_name='nomad_archive.zip', compress=compress)
elif res_type == 'json':
archive_data = to_json(generator())
code_snippet = build_snippet(args, os.path.join(api.base_url, ns.name, 'query'))
data = {'archive_data': archive_data, 'code_snippet': code_snippet}
return data, 200
else:
raise Exception('Unknown res_type %s' % res_type)
@ns.route('/metainfo/<string:metainfo_package_name>')
......
......@@ -16,10 +16,11 @@
Common data, variables, decorators, models used throughout the API.
"""
from typing import Callable, IO, Set, Tuple, Iterable
from flask_restplus import fields
from flask_restplus import fields, abort
import zipstream
from flask import stream_with_context, Response
import sys
import json
from nomad.app.utils import RFC3339DateTime
from nomad.files import Restricted
......@@ -153,3 +154,42 @@ def streamed_zipfile(
response = Response(stream_with_context(generator()), mimetype='application/zip')
response.headers['Content-Disposition'] = 'attachment; filename={}'.format(zipfile_name)
return response
def to_json(files: Iterable[Tuple[str, str, Callable[[str], IO], Callable[[str], int]]]):
data = {}
for _, file_id, open_io, _ in files:
try:
f = open_io(file_id)
data[file_id] = json.loads(f.read())
except KeyError:
pass
except Restricted:
abort(401, message='Not authorized to access %s.' % file_id)
return data
def build_snippet(args, base_url):
str_code = 'import requests\n'
str_code += 'from urllib.parse import urlencode\n'
str_code += '\n\n'
str_code += 'def query_repository(args, base_url):\n'
str_code += ' url = "%s?%s" % (base_url, urlencode(args))\n'
str_code += ' response = requests.get(url)\n'
str_code += ' if response.status_code != 200:\n'
str_code += ' raise Exception("nomad return status %d" % response.status_code)\n'
str_code += ' return response.json()\n'
str_code += '\n\n'
str_code += 'args = {'
for key, val in args.items():
if val is None:
continue
if isinstance(val, str):
str_code += '"%s": "%s", ' % (key, val)
else:
str_code += '"%s": %s, ' % (key, val)
str_code += '}\n'
str_code += 'base_url = "%s"\n' % base_url
str_code += 'JSON_DATA = query_repository(args, base_url)\n'
return str_code
......@@ -24,6 +24,7 @@ from elasticsearch_dsl import Q
from elasticsearch.exceptions import NotFoundError
import elasticsearch.helpers
from datetime import datetime
import os.path
from nomad import search, utils, datamodel, processing as proc, infrastructure
from nomad.app.utils import rfc3339DateTime, RFC3339DateTime, with_logger
......@@ -32,7 +33,7 @@ from nomad.datamodel import UserMetadata, Dataset, User
from .api import api
from .auth import authenticate
from .common import pagination_model, pagination_request_parser, calc_route
from .common import pagination_model, pagination_request_parser, calc_route, build_snippet
ns = api.namespace('repo', description='Access repository metadata.')
......@@ -80,6 +81,8 @@ repo_calcs_model_fields = {
'value and quantity value as key. The possible metrics are code runs(calcs), %s. '
'There is a pseudo quantity "total" with a single value "all" that contains the '
' metrics over all results. ' % ', '.join(datamodel.Domain.instance.metrics_names))),
'code_snippet': fields.String(description=(
'A string of python code snippet which can be executed to reproduce the api result.')),
}
for group_name, (group_quantity, _) in search.groups.items():
repo_calcs_model_fields[group_name] = fields.Nested(api.model('RepoDatasets', {
......@@ -300,6 +303,10 @@ class RepoCalcsResource(Resource):
if args.get(group_name, False):
results[group_name] = quantities[group_quantity]
# build python code snippet
snippet = build_snippet(args, os.path.join(api.base_url, ns.name, ''))
results['code_snippet'] = snippet
return results, 200
except search.ScrollIdNotFound:
abort(400, 'The given scroll_id does not exist.')
......
......@@ -667,6 +667,16 @@ class TestArchive(UploadFilesBasedTests):
assert rv.status_code == 200
assert_zip_file(rv, files=1)
def test_code_snippet(self, api, processeds, test_user_auth):
query_params = {'atoms': 'Si', 'res_type': 'json'}
url = '/archive/query?%s' % urlencode(query_params)
rv = api.get(url, headers=test_user_auth)
assert rv.status_code == 200
data = json.loads(rv.data)
assert isinstance(data, dict)
assert data['code_snippet'] is not None
class TestRepo():
@pytest.fixture(scope='class')
......@@ -1058,6 +1068,14 @@ class TestRepo():
data = json.loads(rv.data)
assert data['pagination']['total'] > 0
def test_code_snippet(self, api, example_elastic_calcs, test_user_auth):
rv = api.get('/repo/?per_page=10', headers=test_user_auth)
assert rv.status_code == 200
data = json.loads(rv.data)
assert data['code_snippet'] is not None
# exec does not seem to work
# exec(data['code_snippet'])
class TestEditRepo():
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment