Skip to content

GitLab

Commit e42f03ae authored by Lauri Himanen's avatar Lauri Himanen
Browse files

Merged.

parents 17541e25 e893fbff
Pipeline #77853 passed with stages
in 28 minutes and 43 seconds
Expand all Hide whitespace changes
Inline Side-by-side
tinker @ 85cc99b3
Compare af2bcff3...85cc99b3
Subproject commit af2bcff3956b83698e3c3fa733df7b5a54264ee1
Subproject commit 85cc99b320454f8c8a68bdf376e128c156cf2ac2
python_common @ 4c83e96b
Compare 2e385c1d...4c83e96b
Subproject commit 2e385c1dbf934157d0e533ee709595fd0ccfb742
Subproject commit 4c83e96bcbf9080d431b7b22f1ecabd4b64dec27
docs/client/install.rst
View file @ e42f03ae
......@@ -5,12 +5,12 @@ We release the NOMAD client library as a Python `distutils <https://docs.python.
You can download and install it the usual way using *pip* (or *conda*).
Install from pypi
.. parsed-literal::
.. code-block:: sh
pip install nomad-lab
Download and install latest release from nomad
.. parsed-literal::
.. code-block:: sh
curl https://repository.nomad-coe.eu/v0.8/dist/nomad-lab.tar.gz -o nomad-lab.tar.gz
pip install ./nomad-lab.tar.gz
......@@ -24,14 +24,16 @@ Other functions, e.g. using the NOMAD parsers to parse your code output, require
additional dependencies. You can use the ``[extra]`` notation to install these extra
requirements:
.. parsed-literal::
.. code-block:: sh
pip install nomad-lab[parsing]
pip install nomad-lab[infrastructure]
pip install nomad-lab[dev]
pip install nomad-lab[all]
The various *extras* have the following meaning:
- ``parsing``, everything necessary to run the parsers
- ``infrastructure``, everything to run NOMAD services
- ``dev``, additional tools that are necessary to develop NOMAD
- ``all``, all of the above
docs/dev/setup.md
View file @ e42f03ae
......@@ -80,7 +80,14 @@ To install libmagick for conda, you can use (other channels might also work):
conda -c conda-forge install --name nomad_env libmagic
```
The next steps can be done using the `setup.sh` script. If you prefere to understand all
#### pip
Make sure you have the most recent version of pip:
```
pip install --upgrade pip
```
The next steps can be done using the `setup.sh` script. If you prefer to understand all
the steps and run them manually, read on:
......
gui/public/env.js
View file @ e42f03ae
......@@ -4,7 +4,13 @@ window.nomadEnv = {
'keycloakClientId': 'nomad_gui_dev',
'appBase': 'http://localhost:8000/fairdi/nomad/latest',
'debug': false,
'sendTrackingData': true,
'matomoEnabled': true,
'matomoUrl': 'https://repository.nomad-coe.eu/fairdi/stat',
'matomoSiteId': '2'
'matomoSiteId': '2',
'version': {
"label": "0.8.1",
"isBeta": true,
"usesBetaData": true,
"officialUrl": "https://repository.nomad-coe.eu/app/gui"
}
}
gui/src/components/About.js
View file @ e42f03ae
......@@ -99,6 +99,9 @@ export default function About() {
makeClickable('encyclopedia', () => {
window.location.href = 'https://encyclopedia.nomad-coe.eu/gui/#/search'
})
makeClickable('analytics', () => {
window.location.href = 'https://www.nomad-coe.eu/index.php?page=bigdata-analyticstoolkit'
})
makeClickable('search', () => {
history.push('/search')
})
......@@ -123,8 +126,8 @@ export default function About() {
}
}
setText('repositoryStats', [
value('n_entries', 'entries'),
value('n_uploads', 'uploads')
value('n_entries', 'entries')
// value('n_uploads', 'uploads')
])
setText('archiveStats', [
value('n_calculations', 'results'),
......
gui/src/components/App.js
View file @ e42f03ae
// trigger rebuild
import React, { useEffect, useState, useContext, useCallback, useRef } from 'react'
import PropTypes, { instanceOf } from 'prop-types'
import PropTypes from 'prop-types'
import { compose } from 'recompose'
import classNames from 'classnames'
import { MuiThemeProvider, withStyles, makeStyles } from '@material-ui/core/styles'
import { LinearProgress, MenuList, Typography,
AppBar, Toolbar, Button, DialogContent, DialogTitle, DialogActions, Dialog, Tooltip,
Snackbar, SnackbarContent } from '@material-ui/core'
Snackbar, SnackbarContent, FormGroup, FormControlLabel, Switch, IconButton } from '@material-ui/core'
import { Route, Link, withRouter, useLocation } from 'react-router-dom'
import BackupIcon from '@material-ui/icons/Backup'
import SearchIcon from '@material-ui/icons/Search'
......@@ -17,6 +15,8 @@ import FAQIcon from '@material-ui/icons/QuestionAnswer'
import MetainfoIcon from '@material-ui/icons/Info'
import DocIcon from '@material-ui/icons/Help'
import CodeIcon from '@material-ui/icons/Code'
import TermsIcon from '@material-ui/icons/Assignment'
import UnderstoodIcon from '@material-ui/icons/Check'
import {help as searchHelp, default as SearchPage} from './search/SearchPage'
import HelpDialog from './Help'
import { ApiProvider, withApi, apiContext } from './api'
......@@ -24,11 +24,9 @@ import { ErrorSnacks, withErrors } from './errors'
import { help as entryHelp, default as EntryPage } from './entry/EntryPage'
import About from './About'
import LoginLogout from './LoginLogout'
import { guiBase, consent, nomadTheme, appBase } from '../config'
import { guiBase, consent, nomadTheme, appBase, version } from '../config'
import {help as metainfoHelp, default as MetaInfoBrowser} from './metaInfoBrowser/MetaInfoBrowser'
import packageJson from '../../package.json'
import { Cookies, withCookies } from 'react-cookie'
import Markdown from './Markdown'
import {help as uploadHelp, default as UploadPage} from './uploads/UploadPage'
import ResolvePID from './entry/ResolvePID'
import DatasetPage from './DatasetPage'
......@@ -37,6 +35,9 @@ import {help as userdataHelp, default as UserdataPage} from './UserdataPage'
import ResolveDOI from './dataset/ResolveDOI'
import FAQ from './FAQ'
import EntryQuery from './entry/EntryQuery'
import {matomo} from '../index'
import { useCookies } from 'react-cookie'
import Markdown from './Markdown'
export const ScrollContext = React.createContext({scrollParentRef: null})
......@@ -80,7 +81,7 @@ const useMainMenuItemStyles = makeStyles(theme => ({
}
}))
function MainMenuItem({tooltip, title, path, href, icon}) {
function MainMenuItem({tooltip, title, path, href, onClick, icon}) {
const {pathname} = useLocation()
const classes = useMainMenuItemStyles()
const selected = path === pathname || (path !== '/' && pathname.startsWith(path))
......@@ -91,6 +92,7 @@ function MainMenuItem({tooltip, title, path, href, icon}) {
color={selected ? 'primary' : 'default'}
size="small"
startIcon={icon}
onClick={onClick}
{...rest}
>
{title}
......@@ -102,9 +104,115 @@ MainMenuItem.propTypes = {
'title': PropTypes.string.isRequired,
'path': PropTypes.string,
'href': PropTypes.string,
'onClick': PropTypes.func,
'icon': PropTypes.element.isRequired
}
const useBetaSnackStyles = makeStyles(theme => ({
root: {},
snack: {
backgroundColor: amber[700]
}
}))
function BetaSnack() {
const classes = useBetaSnackStyles()
const [understood, setUnderstood] = useState(false)
if (!version) {
console.log.warning('no version data available')
return ''
}
if (!version.isBeta) {
return ''
}
return <Snackbar className={classes.root}
anchorOrigin={{
vertical: 'bottom',
horizontal: 'left'
}}
open={!understood}
>
<SnackbarContent
className={classes.snack}
message={<span style={{color: 'white'}}>
You are using a beta version of NOMAD ({version.label}). {
version.usesBetaData ? 'This version is not using the official data. Everything you upload here, might get lost.' : ''
} Click <a style={{color: 'white'}} href={version.officialUrl}>here for the official NOMAD version</a>.
</span>}
action={[
<IconButton key={0} color="inherit" onClick={() => setUnderstood(true)}>
<UnderstoodIcon />
</IconButton>
]}
/>
</Snackbar>
}
function Consent() {
const [cookies, setCookie] = useCookies()
const [accepted, setAccepted] = useState(cookies['terms-accepted'])
const [optOut, setOptOut] = useState(cookies['tracking-enabled'] === 'false')
useEffect(() => {
if (!optOut) {
matomo.push(['setConsentGiven'])
} else {
matomo.push(['requireConsent'])
}
})
const handleClosed = accepted => {
if (accepted) {
setCookie('terms-accepted', true)
setCookie('tracking-enabled', !optOut)
setAccepted(true)
}
}
const handleOpen = () => {
setCookie('terms-accepted', false)
setAccepted(false)
}
return (
<React.Fragment>
<MainMenuItem
title="Terms"
onClick={handleOpen}
tooltip="NOMAD's terms"
icon={<TermsIcon/>}
/>
<Dialog
disableBackdropClick disableEscapeKeyDown
open={!accepted}
>
<DialogTitle>Terms of Use</DialogTitle>
<DialogContent>
<Markdown>{consent}</Markdown>
<FormGroup>
<FormControlLabel
control={<Switch
checked={optOut}
onChange={(e) => {
setOptOut(!optOut)
}}
color="primary"
/>}
label="Do not provide information about your use of NOMAD (opt-out)."
/>
</FormGroup>
</DialogContent>
<DialogActions>
<Button onClick={() => handleClosed(true)} color="primary">
Accept
</Button>
</DialogActions>
</Dialog>
</React.Fragment>
)
}
const useMainMenuStyles = makeStyles(theme => ({
root: {
display: 'inline-flex',
......@@ -186,6 +294,7 @@ function MainMenu() {
tooltip="NOMAD's Gitlab project"
icon={<CodeIcon/>}
/>
<Consent />
</MenuList>
}
......@@ -378,62 +487,6 @@ class NavigationUnstyled extends React.Component {
const Navigation = compose(withRouter, withErrors, withApi(false), withStyles(NavigationUnstyled.styles))(NavigationUnstyled)
class LicenseAgreementUnstyled extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
cookies: instanceOf(Cookies).isRequired
}
static styles = theme => ({
content: {
backgroundColor: theme.palette.primary.main
},
button: {
color: 'white'
}
})
constructor(props) {
super(props)
this.handleClosed = this.handleClosed.bind(this)
}
state = {
accepted: this.props.cookies.get('terms-accepted')
}
handleClosed(accepted) {
if (accepted) {
this.props.cookies.set('terms-accepted', true)
this.setState({accepted: true})
}
}
render() {
return (
<div>
<Dialog
disableBackdropClick disableEscapeKeyDown
open={!this.state.accepted}
>
<DialogTitle>Terms of Use</DialogTitle>
<DialogContent>
<Markdown>{consent}</Markdown>
</DialogContent>
<DialogActions>
<Button onClick={() => this.handleClosed(true)} color="primary">
Accept
</Button>
</DialogActions>
</Dialog>
</div>
)
}
}
const LicenseAgreement = compose(withCookies, withStyles(LicenseAgreementUnstyled.styles))(LicenseAgreementUnstyled)
const routes = {
'about': {
exact: true,
......@@ -493,6 +546,7 @@ class App extends React.PureComponent {
render() {
return (
<MuiThemeProvider theme={nomadTheme}>
<BetaSnack />
<ErrorSnacks>
<ApiProvider>
<Navigation>
......@@ -518,7 +572,6 @@ class App extends React.PureComponent {
</Navigation>
</ApiProvider>
</ErrorSnacks>
<LicenseAgreement />
</MuiThemeProvider>
)
}
......
gui/src/components/about.svg
View file @ e42f03ae
This diff is collapsed.
gui/src/components/domains.js
View file @ e42f03ae
......@@ -114,7 +114,7 @@ export const domains = ({
supportsSort: true
},
'dft.xc_functional': {
label: 'XT treatment',
label: 'XC functionals',
supportsSort: true
},
'dft.system': {
......
gui/src/config.js
View file @ e42f03ae
import { createMuiTheme } from '@material-ui/core'
window.nomadEnv = window.nomadEnv || {}
export const version = window.nomadEnv.version
export const appBase = window.nomadEnv.appBase.replace(/\/$/, '')
// export const apiBase = 'http://repository.nomad-coe.eu/v0.8/api'
export const apiBase = `${appBase}/api`
......@@ -12,7 +13,7 @@ export const keycloakBase = window.nomadEnv.keycloakBase
export const keycloakRealm = window.nomadEnv.keycloakRealm
export const keycloakClientId = window.nomadEnv.keycloakClientId
export const debug = window.nomadEnv.debug || false
export const sendTrackingData = window.nomadEnv.sendTrackingData
export const matomoEnabled = window.nomadEnv.matomoEnabled
export const email = 'webmaster@nomad-coe.eu'
export const maxLogsToShow = 50
......@@ -27,8 +28,10 @@ you and users you share your data with. The *embargo period* lasts up to 36 mont
After the *embargo* your published data will be public. **Note that public data
is visible to others and files become downloadable by everyone.**
This web-site uses *cookies*. By using this web-site you agree to our use
of *cookies*. [Learn more](https://www.cookiesandyou.com/).
This web-site uses *cookies*. We use cookies to track you login status for all NOMAD services
and optionally to store information about your use of NOMAD. None of this information is
shared with other parties. By using this web-site you agree to the described use of *cookies*.
[Learn more](https://www.cookiesandyou.com/).
`
export const nomadPrimaryColor = {
main: '#008DC3',
......
gui/src/index.js
View file @ e42f03ae
......@@ -8,16 +8,16 @@ import { Router, Route } from 'react-router-dom'
import { QueryParamProvider } from 'use-query-params'
import history from './history'
import PiwikReactRouter from 'piwik-react-router'
import { sendTrackingData, matomoUrl, matomoSiteId, keycloakBase, keycloakRealm, keycloakClientId } from './config'
import { matomoEnabled, matomoUrl, matomoSiteId, keycloakBase, keycloakRealm, keycloakClientId } from './config'
import Keycloak from 'keycloak-js'
import { KeycloakProvider } from 'react-keycloak'
import * as serviceWorker from './serviceWorker'
const matomo = sendTrackingData ? PiwikReactRouter({
export const matomo = matomoEnabled ? PiwikReactRouter({
url: matomoUrl,
siteId: matomoSiteId,
clientTrackerName: 'stat.js',
serverTrackerName: 'stat.php'
serverTrackerName: 'stat'
}) : null
const keycloak = Keycloak({
......@@ -26,9 +26,11 @@ const keycloak = Keycloak({
clientId: keycloakClientId
})
// matomo.push('requireConsent')
ReactDOM.render(
<KeycloakProvider keycloak={keycloak} initConfig={{onLoad: 'check-sso'}} LoadingComponent={<div />}>
<Router history={sendTrackingData ? matomo.connectToHistory(history) : history}>
<Router history={matomoEnabled ? matomo.connectToHistory(history) : history}>
<QueryParamProvider ReactRouterRoute={Route}>
<App />
</QueryParamProvider>
......
nomad/app/api/common.py
View file @ e42f03ae
......@@ -201,7 +201,8 @@ def apply_search_parameters(search_request: search.SearchRequest, args: Dict[str
try:
optimade = args.get('dft.optimade', None)
if optimade is not None:
q = filterparser.parse_filter(optimade)
q = filterparser.parse_filter(
optimade, nomad_properties=domain, without_prefix=True)
search_request.query(q)
except filterparser.FilterException as e:
abort(400, 'Could not parse optimade query: %s' % (str(e)))
......@@ -339,18 +340,22 @@ def query_api_clientlib(**kwargs):
return value
kwargs = {
query = {
key: normalize_value(key, value) for key, value in kwargs.items()
if key in search.search_quantities and (key != 'domain' or value != config.meta.default_domain)
}
for key in ['dft.optimade']:
if key in kwargs:
query[key] = kwargs[key]
out = io.StringIO()
out.write('from nomad import client, config\n')
out.write('config.client.url = \'%s\'\n' % config.api_url(ssl=False))
out.write('results = client.query_archive(query={%s' % ('' if len(kwargs) == 0 else '\n'))
out.write(',\n'.join([
' \'%s\': %s' % (key, pprint.pformat(value, compact=True))
for key, value in kwargs.items()]))
for key, value in query.items()]))
out.write('})\n')
out.write('print(results)\n')
......
nomad/app/optimade/endpoints.py
View file @ e42f03ae
......@@ -22,10 +22,10 @@ from nomad.datamodel import OptimadeEntry
from .api import api, url, base_request_args
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
Links as LinksModel, CalculationDataObject, single_entry_endpoint_parser, base_endpoint_parser, \
json_api_info_response_model, json_api_list_response_model, StructureObject, \
ToplevelLinks, \
json_api_structure_response_model, json_api_structures_response_model
Links as LinksModel, single_entry_endpoint_parser, base_endpoint_parser, \
json_api_info_response_model, json_api_list_response_model, EntryDataObject, \
ToplevelLinks, get_entry_properties, json_api_structure_response_model, \
json_api_structures_response_model
from .filterparser import parse_filter, FilterException
ns = api.namespace('v0', description='The version v0 API namespace with all OPTiMaDe endpoints.')
......@@ -100,7 +100,7 @@ class CalculationList(Resource):
available = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
assert len(results) == len(result['results']), 'archive and elasticsearch are not consistent'
return dict(
meta=Meta(
......@@ -114,7 +114,7 @@ class CalculationList(Resource):
page_number=page_number,
page_limit=page_limit,
sort=sort, filter=filter),
data=[CalculationDataObject(d, request_fields=request_fields) for d in results]
data=[EntryDataObject(d, optimade_type='calculations', request_fields=request_fields) for d in results]
), 200
......@@ -143,7 +143,7 @@ class Calculation(Resource):
return dict(
meta=Meta(query=request.url, returned=1),
data=CalculationDataObject(results[0], request_fields=request_fields)
data=EntryDataObject(results[0], optimade_type='calculations', request_fields=request_fields)
), 200
......@@ -159,9 +159,7 @@ class CalculationInfo(Resource):
result = {
'description': 'a calculation entry',
'properties': {
attr.name: dict(description=attr.description)
for attr in OptimadeEntry.m_def.all_properties.values()},
'properties': get_entry_properties(),
'formats': ['json'],
'output_fields_by_format': {
'json': list(OptimadeEntry.m_def.all_properties.keys())}
......@@ -343,7 +341,7 @@ class StructureList(Resource):
page_limit=page_limit,
sort=sort, filter=filter
),
data=[StructureObject(d, request_fields) for d in results]
data=[EntryDataObject(d, optimade_type='structures', request_fields=request_fields) for d in results]
), 200
......@@ -372,7 +370,7 @@ class Structure(Resource):
return dict(
meta=Meta(query=request.url, returned=1),
data=StructureObject(results[0], request_fields=request_fields)
data=EntryDataObject(results[0], optimade_type='structures', request_fields=request_fields)
), 200
......@@ -388,9 +386,7 @@ class StructuresInfo(Resource):
result = {
'description': 'a structure entry',
'properties': {
attr.name: dict(description=attr.description)
for attr in OptimadeEntry.m_def.all_properties.values()},
'properties': get_entry_properties(),
'formats': ['json'],
'output_fields_by_format': {
'json': list(OptimadeEntry.m_def.all_properties.keys())}
......
nomad/app/optimade/filterparser.py
View file @ e42f03ae
......@@ -14,50 +14,75 @@
from typing import Dict
from elasticsearch_dsl import Q
from cachetools import cached
from optimade.filterparser import LarkParser
from optimade.filtertransformers.elasticsearch import ElasticTransformer, Quantity
from nomad.search import search_quantities
_parser = LarkParser(version=(0, 10, 1))
class FilterException(Exception):
''' Raised on parsing a filter expression with syntactic of semantic errors. '''
pass
_quantities: Dict[str, Quantity] = None
_parser = LarkParser(version=(0, 10, 1))
_transformer = None
@cached(cache={})
def _get_transformer(nomad_properties, without_prefix):
from nomad.datamodel import OptimadeEntry
quantities: Dict[str, Quantity] = {
q.name: Quantity(
q.name, es_field='dft.optimade.%s' % q.name,
elastic_mapping_type=q.a_search.mapping.__class__)
for q in OptimadeEntry.m_def.all_quantities.values()
if 'search' in q.m_annotations}
quantities['elements'].length_quantity = quantities['nelements']
quantities['dimension_types'].length_quantity = quantities['dimension_types']
quantities['elements'].has_only_quantity = Quantity(name='only_atoms')
quantities['elements'].nested_quantity = quantities['elements_ratios']
quantities['elements_ratios'].nested_quantity = quantities['elements_ratios']
if nomad_properties is not None:
for search_quantity in search_quantities.values():
name = search_quantity.name
if '.' in name:
if name.startswith(nomad_properties):
name = name[len(nomad_properties) + 1:]
else:
continue
names = ['_nmd_' + name]
if without_prefix:
names.append(name)
for name in names:
if name not in quantities:
quantities[name] = Quantity(