Commit 8030ce7f authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added clientlib code to UI. Fixed cli imports.

parent 12887945
Pipeline #72666 canceled with stages
in 3 minutes and 22 seconds
......@@ -38,6 +38,27 @@ class ApiDialogUnstyled extends React.Component {
}
})
renderCode(title, code) {
const {classes} = this.props
return <React.Fragment>
<Typography>{title}</Typography>
<div className={classes.codeContainer}>
<div className={classes.code}>
<Markdown text={'```\n' + code + '\n```'} />
</div>
<div className={classes.codeActions}>
<CopyToClipboard text={code} onCopy={() => null}>
<Tooltip title="Copy to clipboard">
<IconButton>
<ClipboardIcon />
</IconButton>
</Tooltip>
</CopyToClipboard>
</div>
</div>
</React.Fragment>
}
render() {
const { classes, title, data, onClose, ...dialogProps } = this.props
......@@ -46,45 +67,12 @@ class ApiDialogUnstyled extends React.Component {
<DialogTitle>{title || 'API Code'}</DialogTitle>
<DialogContent classes={{root: classes.content}}>
<Typography>Access the archive as JSON via <i>curl</i>:</Typography>
<div className={classes.codeContainer}>
<div className={classes.code}>
<Markdown>{`
\`\`\`
${data.curl}
\`\`\`
`}</Markdown>
</div>
<div className={classes.codeActions}>
<CopyToClipboard text={data.curl} onCopy={() => null}>
<Tooltip title="Copy to clipboard">
<IconButton>
<ClipboardIcon />
</IconButton>
</Tooltip>
</CopyToClipboard>
</div>
</div>
<Typography>Access the archive in <i>python</i>:</Typography>
<div className={classes.codeContainer}>
<div className={classes.code}>
<Markdown>{`
\`\`\`
${data.python}
\`\`\`
`}</Markdown>
</div>
<div className={classes.codeActions}>
<CopyToClipboard text={data.python} onCopy={() => null}>
<Tooltip title="Copy to clipboard">
<IconButton>
<ClipboardIcon />
</IconButton>
</Tooltip>
</CopyToClipboard>
</div>
</div>
{ data.code && data.code.curl &&
this.renderCode(<span>Access the archive as JSON via <i>curl</i>:</span>, data.code.curl)}
{ data.code && data.code.python &&
this.renderCode(<span>Access the archive in <i>python</i>:</span>, data.code.python)}
{ data.code && data.code.clientlib &&
this.renderCode(<span>Access the archive with the <i>NOMAD client library</i>:</span>, data.code.clientlib)}
<Typography>The repository API response as JSON:</Typography>
<div className={classes.codeContainer}>
......
......@@ -25,9 +25,9 @@ from werkzeug.wsgi import DispatcherMiddleware # pylint: disable=E0611
import os.path
import random
from structlog import BoundLogger
import orjson
import collections
from mongoengine.base.datastructures import BaseList
import orjson
from nomad import config, utils as nomad_utils
......@@ -39,9 +39,24 @@ from .gui import blueprint as gui_blueprint
from . import common
def dump_json(data):
def default(data):
if isinstance(data, collections.OrderedDict):
return dict(data)
if data.__class__.__name__ == 'BaseList':
return list(data)
raise TypeError
return orjson.dumps(
data, default=default,
option=orjson.OPT_INDENT_2 | orjson.OPT_NON_STR_KEYS)
# replace the json implementation of flask_restplus
def output_json(data, code, headers=None):
dumped = nomad_utils.dumps(data) + b'\n'
dumped = dump_json(data) + b'\n'
resp = make_response(dumped, code)
resp.headers.extend(headers or {})
......
......@@ -20,6 +20,8 @@ from flask_restplus import fields
import zipstream
from flask import stream_with_context, Response, g, abort
from urllib.parse import urlencode
import pprint
import io
import sys
import os.path
......@@ -70,10 +72,14 @@ search_model_fields = {
'results': fields.List(fields.Raw(allow_null=True, skip_none=True), description=(
'A list of search results. Each result is a dict with quantitie names as key and '
'values as values'), allow_null=True, skip_none=True),
'python': fields.String(description=(
'A string of python code snippet which can be executed to reproduce the api result.')),
'curl': fields.String(description=(
'A string of curl command which can be executed to reproduce the api result.'))}
'code': fields.Nested(api.model('Code', {
'python': fields.String(description=(
'A piece of python code snippet which can be executed to reproduce the api result.')),
'curl': fields.String(description=(
'A curl command which can be executed to reproduce the api result.')),
'clientlib': fields.String(description=(
'A piece of python code which uses NOMAD\'s client library to access the archive.'))
}), allow_null=True, skip_none=True)}
search_model = api.model('Search', search_model_fields)
......@@ -307,6 +313,30 @@ response = requests.post("{}")
data = response.json()'''.format(url)
def query_api_clientlib(**kwargs):
'''
Creates a string of python code to execute a search query on the archive using
the client library.
'''
kwargs = {
key: value for key, value in kwargs.items()
if key in _search_quantities and (key != 'domain' or value != [config.default_domain])
}
out = io.StringIO()
out.write('from nomad import client, config\n')
out.write('config.client.url = \'%s\'\n' % config.api_url(ssl=False))
out.write('results = client.query_archive(query={%s' % ('' if len(kwargs) == 0 else '\n'))
out.write(',\n'.join([
' \'%s\': %s' % (key, pprint.pformat(value, compact=True))
for key, value in kwargs.items()]))
out.write('})\n')
out.write('print(results)\n')
print(out.getvalue())
return out.getvalue()
def query_api_curl(*args, **kwargs):
'''
Creates a string of curl command to execute a search query to the repository.
......
......@@ -35,7 +35,7 @@ from .api import api
from .auth import authenticate
from .common import search_model, calc_route, add_pagination_parameters,\
add_scroll_parameters, add_search_parameters, apply_search_parameters,\
query_api_python, query_api_curl, _search_quantities
query_api_python, query_api_curl, query_api_clientlib, _search_quantities
ns = api.namespace('repo', description='Access repository metadata.')
......@@ -67,8 +67,11 @@ class RepoCalcResource(Resource):
abort(401, message='Not authorized to access %s/%s.' % (upload_id, calc_id))
result = calc.to_dict()
result['python'] = query_api_python('archive', upload_id, calc_id)
result['curl'] = query_api_curl('archive', upload_id, calc_id)
result['code'] = {
'python': query_api_python('archive', upload_id, calc_id),
'curl': query_api_curl('archive', upload_id, calc_id),
'clientlib': query_api_clientlib(upload_id=[upload_id], calc_id=[calc_id])
}
return result, 200
......@@ -251,8 +254,11 @@ class RepoCalcsResource(Resource):
code_args = dict(request.args)
if 'statistics' in code_args:
del(code_args['statistics'])
results['curl'] = query_api_curl('archive', 'query', query_string=code_args)
results['python'] = query_api_python('archive', 'query', query_string=code_args)
results['code'] = {
'curl': query_api_curl('archive', 'query', query_string=code_args),
'python': query_api_python('archive', 'query', query_string=code_args),
'clientlib': query_api_clientlib(**code_args)
}
return results, 200
except search.ScrollIdNotFound:
......
......@@ -18,6 +18,11 @@ that offers various functionality to the command line user.
Use it from the command line with ``nomad --help`` or ``python -m nomad.cli --help`` to learn
more.
The CLI uses lazy_import for lazy loading modules. This has some limitations. You will
break lazy loading if an ``from x import y`` is used in the cli code. You will also
have to add imports via :func:`nomad.cli.lazy_import.lazy_module` before importing
them.
'''
from nomad.cli import lazy_import
......@@ -26,7 +31,6 @@ lazy_import.lazy_module('logging')
lazy_import.lazy_module('os')
lazy_import.lazy_module('typing')
lazy_import.lazy_module('json')
lazy_import.lazy_module('orjson')
lazy_import.lazy_module('sys')
lazy_import.lazy_module('nomad.config')
lazy_import.lazy_module('nomad.infrastructure')
......@@ -36,6 +40,7 @@ lazy_import.lazy_module('nomad.normalizing')
lazy_import.lazy_module('nomad.datamodel')
lazy_import.lazy_module('nomad.search')
lazy_import.lazy_module('nomad.metainfo')
lazy_import.lazy_module('nomad.atomutils')
lazy_import.lazy_module('nomad.processing')
lazy_import.lazy_module('nomad.client')
lazy_import.lazy_module('nomadcore')
......
......@@ -40,6 +40,7 @@ lazy_import.lazy_module('bs4')
lazy_import.lazy_module('matid')
lazy_import.lazy_module('matid.symmetry.symmetryanalyzer')
lazy_import.lazy_module('matid.utils.segfault_protect')
lazy_import.lazy_module('nomad.atomutils')
lazy_import.lazy_module('nomad.normalizing')
lazy_import.lazy_module('nomad.processing')
lazy_import.lazy_module('nomad.search')
......
......@@ -27,7 +27,7 @@ import numpy as np
import requests
import ase
import bs4
from matid import SymmetryAnalyzer
import matid
from nomad import processing as proc, search, datamodel, infrastructure, utils, config
from nomad import atomutils
......@@ -490,7 +490,7 @@ def prototypes_update(ctx, filepath, matches_only):
# Try to first see if the space group can be matched with the one in AFLOW
try:
symm = SymmetryAnalyzer(atoms, config.normalize.prototype_symmetry_tolerance)
symm = matid.SymmetryAnalyzer(atoms, config.normalize.prototype_symmetry_tolerance)
spg_number = symm.get_space_group_number()
wyckoff_matid = symm.get_wyckoff_letters_conventional()
norm_system = symm.get_conventional_system()
......
......@@ -38,7 +38,6 @@ from collections import OrderedDict
import base64
from contextlib import contextmanager
import json
import orjson
import uuid
import time
import re
......@@ -105,21 +104,6 @@ def set_console_log_level(level):
handler.setLevel(level)
def dumps(data):
def default(data):
if isinstance(data, collections.OrderedDict):
return dict(data)
if data.__class__.__name__ == 'BaseList':
return list(data)
raise TypeError
return orjson.dumps(
data, default=default,
option=orjson.OPT_INDENT_2 | orjson.OPT_NON_STR_KEYS)
def decode_handle_id(handle_str: str):
result = 0
for c in handle_str:
......
......@@ -776,8 +776,9 @@ class TestRepo():
rv = api.get('/repo/0/1', headers=test_user_auth)
assert rv.status_code == 200
data = rv.json
assert data['python'] is not None
assert data['curl'] is not None
assert data['code']['curl'] is not None
assert data['code']['python'] is not None
assert data['code']['clientlib'] is not None
def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
rv = api.get('/repo/0/1', headers=other_test_user_auth)
......@@ -1159,8 +1160,9 @@ class TestRepo():
rv = api.get('/repo/?code_name=VASP', headers=test_user_auth)
assert rv.status_code == 200
data = json.loads(rv.data)
assert data['python'] is not None
assert data['curl'] is not None
assert data['code']['curl'] is not None
assert data['code']['python'] is not None
assert data['code']['clientlib'] is not None
class TestEditRepo():
......
......@@ -14,8 +14,9 @@
import ase.build
from nomad import datamodel, config, utils
from nomad import datamodel, config
from nomad.parsing import Backend
from nomad.app import dump_json
from tests.test_parsing import parsed_vasp_example # pylint: disable=unused-import
from tests.test_parsing import parsed_template_example # pylint: disable=unused-import
......@@ -103,7 +104,8 @@ def assert_normalized(backend: Backend):
assert metadata[key] != config.services.unavailable_value, '%s must not be unavailable' % key
utils.dumps(backend.entry_archive.m_to_dict())
# check if the result can be dumped
dump_json(backend.entry_archive.m_to_dict())
def test_normalizer(normalized_example: Backend):
......
......@@ -18,6 +18,7 @@ import click.testing
import json
import datetime
import zipfile
import time
from nomad import search, processing as proc, files
from nomad.cli import cli
......@@ -29,6 +30,17 @@ from tests.app.test_app import BlueprintClient
# TODO there is much more to test
@pytest.mark.usefixtures('reset_config', 'nomad_logging')
class TestCli:
def test_help(self, example_mainfile):
start = time.time()
result = click.testing.CliRunner().invoke(
cli, ['--help'], catch_exceptions=False)
assert result.exit_code == 0
assert time.time() - start < 1
@pytest.mark.usefixtures('reset_config', 'nomad_logging')
class TestParse:
def test_parser(self, example_mainfile):
......@@ -45,9 +57,6 @@ class TestAdmin:
cli, ['admin', 'reset', '--i-am-really-sure'], catch_exceptions=False)
assert result.exit_code == 0
# allow other test to re-establish a connection
# mongoengine.disconnect_all()
def test_reset_not_sure(self):
result = click.testing.CliRunner().invoke(
cli, ['admin', 'reset'], catch_exceptions=False)
......
......@@ -21,6 +21,7 @@ from shutil import copyfile
from nomad import utils, files, datamodel
from nomad.parsing import parser_dict, match_parser, BrokenParser, BadContextUri, Backend
from nomad.app import dump_json
parser_examples = [
......@@ -402,7 +403,8 @@ def parser_in_dir(dir):
try:
backend = parser.run(file_path)
utils.dumps(backend.entry_archive.m_to_dict())
# check if the result can be dumped
dump_json(backend.entry_archive.m_to_dict())
backend.resource.unload()
except Exception as e:
print(file_path, parser, 'FAILURE', e)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment