Commit 1995952d authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.8.4' into 'master'

V0.8.4

See merge request !139
parents cc540ed5 8d9744c5
Pipeline #79921 passed with stages
in 20 minutes and 15 seconds
'''
A simple example that uses the NOMAD client library to access the archive.
'''
from nomad import config
from nomad.client import ArchiveQuery
from nomad.metainfo import units
......
'''
In this example, we go through many uploads in parallel to extract information from
certain calculations.
The motivation behind this is that the selective access of sections in an archvie might
be slow. If something is read from almost all calculations, it might be faster to
sequentially read all of the upload's archive file.
This is not an API example, but directly accesses archive files. Specifically, we
try to read all fingerprints for upload/calc/material combinations read from an
input file. The fingerprint data gets writting to an output file.
'''
from typing import Any
from multiprocessing import Pool, Queue, Event
from queue import Empty
import json
import traceback
from nomad import files
def read_archive(entries):
try:
upload_id = entries[0]['upload_id']
upload_files = files.UploadFiles.get(upload_id, lambda *args: True)
assert upload_files is not None
for entry in entries:
calc_id = entry['calc_id']
material_id = entry['material_id']
with upload_files.read_archive(calc_id) as archive:
entry_archive = archive[calc_id].to_dict()
for run in entry_archive.get('section_run', []):
for calc in run.get('section_single_configuration_calculation', []):
for dos in calc.get('section_dos', []):
fingerprint = dos.get('section_dos_fingerprint')
if fingerprint:
yield {
'upload_id': upload_id,
'calc_id': calc_id,
'material_id': material_id,
'fingerprint': fingerprint}
except Exception:
traceback.print_exc()
nworker = 24
entry_queue: Any = Queue(maxsize=100)
result_queue: Any = Queue(maxsize=100)
producer_end = Event()
worker_sentinel = 'end'
def worker():
entries = []
while not (producer_end.is_set() and entry_queue.empty()):
try:
entries = entry_queue.get(block=True, timeout=0.1)
except Empty:
continue
for result in read_archive(entries):
result_queue.put(result)
result_queue.put(worker_sentinel)
print('end worker')
def writer():
ended_worker = 0
count = 0
f = open('local/fingerprints.json', 'wt')
f.write('[')
while not (ended_worker == nworker and result_queue.empty()):
try:
result = result_queue.get(block=True, timeout=0.1)
except Empty:
continue
if result == worker_sentinel:
ended_worker += 1
continue
if count > 0:
f.write(',\n')
json.dump(result, f, indent=2)
count += 1
if count % 1000 == 0:
print(count)
f.write(']')
f.close()
print('end writer')
def producer():
with open('local/materials.json', 'r') as f:
data = json.load(f)
upload_id = None
entries = []
for entry in data:
if upload_id is not None and upload_id != entry['upload_id']:
entry_queue.put(entries, block=True)
entries = []
upload_id = entry['upload_id']
entries.append(entry)
entry_queue.put(entries, block=True)
producer_end.set()
print('end producer')
with Pool(processes=nworker + 2) as pool:
for _ in range(nworker):
pool.apply_async(worker)
pool.apply_async(producer)
pool.apply_async(writer)
pool.close()
pool.join()
'''
This examplefies how to send raw queries to elasticsearch.
Specifically this will read all materials with fingerprints from the search engine
and store them in a local file. We use composite aggregations with after-based
pagination.
'''
import json
from nomad import infrastructure, config
infrastructure.setup_files()
infrastructure.setup_elastic()
results = []
after = None
count = 0
while True:
request = {
"query": {
"bool": {
"must": [
{
"match": {
"dft.quantities": "section_dos_fingerprint"
},
},
{
"match": {
"published": True
},
},
{
"match": {
"with_embargo": False
}
}
]
}
},
"size": 0,
"aggs": {
"results": {
"composite": {
"sources": {
"materials": {
"terms": {
"field": "encyclopedia.material.material_id"
}
}
},
"size": 10000
},
"aggs": {
"calcs": {
"top_hits": {
"sort": {
"_script": {
"type": "number",
"script": {
"lang": "painless",
"source": '''
int result = 0;
String code = doc['dft.code_name'].value;
String functional = doc['dft.xc_functional'].value;
if (functional == 'GGA') result += 100;
if (code == 'VASP')
result += 1;
else if (code == 'FHI-aims')
result += 2;
return result;
'''
},
"order": "asc"
},
},
"_source": {
"includes": ['upload_id', 'calc_id', 'dft.code_name', 'dft.xc_functional']
},
"size": 1
}
}
}
}
}
}
if after is not None:
request['aggs']['results']['composite']['after'] = after
res = infrastructure.elastic_client.search(index=config.elastic.index_name, body=request)
if len(res['aggregations']['results']['buckets']) == 0:
break
after = res['aggregations']['results']['after_key']
for material_bucket in res['aggregations']['results']['buckets']:
material_id = material_bucket['key']['materials']
entry = material_bucket['calcs']['hits']['hits'][0]['_source']
upload_id = entry['upload_id']
calc_id = entry['calc_id']
results.append(dict(material_id=material_id, upload_id=upload_id, calc_id=calc_id))
count += 1
print(count)
results.sort(key=lambda item: item['upload_id'])
with open('local/materials.json', 'wt') as f:
f.write(json.dumps(results, indent=2))
import json
from nomad.datamodel.metainfo import public
# A simple example that demonstrates how to set references
run = public.section_run()
scc = run.m_create(public.section_single_configuration_calculation)
system = run.m_create(public.section_system)
scc.single_configuration_calculation_to_system_ref = system
assert scc.single_configuration_calculation_to_system_ref == system
print(json.dumps(run.m_to_dict(), indent=2))
{
"name": "nomad-fair-gui",
"version": "0.8.3",
"version": "0.8.4",
"commit": "e98694e",
"private": true,
"dependencies": {
......
......@@ -5,12 +5,13 @@ window.nomadEnv = {
'appBase': 'http://localhost:8000/fairdi/nomad/latest',
'debug': false,
'matomoEnabled': false,
'matomoUrl': 'https://nomad-lab.eu/prod/stat',
'matomoUrl': 'https://nomad-lab.eu/fairdi/stat',
'matomoSiteId': '2',
'version': {
"label": "0.8.3",
"label": "0.8.4",
"isBeta": false,
"usesBetaData": false,
"officialUrl": "https://nomad-lab.eu/prod/rae/gui"
}
},
'encyclopediaEnabled': true
}
*:focus {
outline: none;
}
body {
overflow-y: hidden;
}
\ No newline at end of file
......@@ -100,7 +100,7 @@ export default function About() {
window.location.href = 'https://encyclopedia.nomad-coe.eu/gui/#/search'
})
makeClickable('analytics', () => {
window.location.href = 'https://nomad-lab.eu/index.php?page=AItutorials'
window.location.href = 'https://nomad-lab.eu/AItutorials'
})
makeClickable('search', () => {
history.push('/search')
......@@ -148,9 +148,9 @@ export default function About() {
If you want to provide your own data, please login or register for an account.
You can learn more about on the NOMAD Repository and Archive
[homepage](https://nomad-lab.eu/index.php?page=repo-arch), our
[homepage](https://nomad-lab.eu/repo-arch), our
[documentation](${appBase}/docs/index.html).
There is also an [FAQ](https://nomad-lab.eu/index.php?page=repository-archive-faqs)
There is also an [FAQ](https://nomad-lab.eu/repository-archive-faqs)
and the more detailed [uploader documentation](${appBase}/docs/upload.html).
`}</Markdown>
</Grid>
......@@ -221,7 +221,7 @@ export default function About() {
There is a [tutorial on how to use the API with plain Python](${appBase}/docs/api_tutorial.html).
Another [tutorial covers how to install and use NOMAD's Python client library](${appBase}/docs/archive_tutorial.html).
The [NOMAD Analytics Toolkit](https://nomad-lab.eu/index.php?page=AIToolkit) allows to use
The [NOMAD Analytics Toolkit](https://nomad-lab.eu/AIToolkit) allows to use
this without installation and directly on NOMAD servers.
`}</Markdown></InfoCard>
<Grid item xs={12}>
......
......@@ -154,6 +154,7 @@ function Consent() {
const [cookies, setCookie] = useCookies()
const [accepted, setAccepted] = useState(cookies['terms-accepted'])
const [optOut, setOptOut] = useState(cookies['tracking-enabled'] === 'false')
const forever = new Date(2147483647 * 1000)
useEffect(() => {
if (!optOut) {
......@@ -165,8 +166,8 @@ function Consent() {
const handleClosed = accepted => {
if (accepted) {
setCookie('terms-accepted', true)
setCookie('tracking-enabled', !optOut)
setCookie('terms-accepted', true, {expires: forever})
setCookie('tracking-enabled', !optOut, {expires: forever})
setAccepted(true)
}
}
......@@ -278,7 +279,7 @@ function MainMenu() {
/>
<MainMenuItem
title="FAQ"
href="https://nomad-lab.eu/index.php?page=repository-archive-faqs"
href="https://nomad-lab.eu/repository-archive-faqs"
tooltip="Frequently Asked Questions (FAQ)"
icon={<FAQIcon/>}
/>
......
......@@ -59,7 +59,6 @@ export default function DatasetPage() {
return <div>loading...</div>
}
console.log('### DatasetPage', dataset)
return <div>
<div className={classes.header}>
<div className={classes.description}>
......
import React from 'react'
import { withApi } from './api'
import Search from './search/Search'
import { encyclopediaEnabled } from '../config'
export const help = `
This page allows you to **inspect** and **manage** you own data. It is similar to the
......@@ -58,7 +59,7 @@ function UserdataPage() {
initialOwner="user"
initialRequest={{order_by: 'upload_time', uploads_grouped: true}}
initialResultTab="uploads"
availableResultTabs={['uploads', 'datasets', 'entries']}
availableResultTabs={['uploads', 'datasets', 'entries', ...(encyclopediaEnabled ? ['materials'] : [])]}
resultListProps={{selectedColumnsKey: 'userEntries', selectedColumns: ['formula', 'upload_time', 'mainfile', 'published', 'co_authors', 'references', 'datasets']}}
/>
}
......
import React from 'react'
import PropTypes from 'prop-types'
import { Typography, Button, makeStyles, Tooltip } from '@material-ui/core'
import { Typography, Tooltip, Link } from '@material-ui/core'
import Quantity from '../Quantity'
import _ from 'lodash'
import {appBase} from '../../config'
const useStyles = makeStyles(theme => ({
actions: {
marginTop: theme.spacing(1),
textAlign: 'right',
margin: -theme.spacing(1)
}
}))
import {appBase, encyclopediaEnabled} from '../../config'
export default function DFTEntryOverview(props) {
const classes = useStyles()
const {data} = props
if (!data.dft) {
return <Typography color="error">No metadata available</Typography>
}
const material_name = entry => entry.encyclopedia.material.material_name
const material_name = entry => {
let name
try {
name = entry.encyclopedia.material.material_name
} catch {}
name = name || 'unnamed'
if (encyclopediaEnabled && data.encyclopedia && data.encyclopedia.material && data.published && !data.with_embargo) {
const url = `${appBase}/encyclopedia/#/material/${data.encyclopedia.material.material_id}`
return (
<Tooltip title="Show the material of this entry in the NOMAD Encyclopedia.">
<Link href={url}>{name}</Link>
</Tooltip>
)
} else {
return name
}
}
return <div>
<Quantity column>
<Quantity row>
<Quantity quantity="formula" label='formula' noWrap {...props} />
<Quantity quantity={material_name} label='material name' noWrap {...props} />
<Quantity quantity={material_name} label='material' noWrap {...props} />
</Quantity>
<Quantity row>
<Quantity quantity="dft.code_name" label='dft code' noWrap {...props} />
......@@ -46,15 +54,6 @@ export default function DFTEntryOverview(props) {
</Quantity>
</Quantity>
</Quantity>
{data.encyclopedia && data.encyclopedia.material &&
<div className={classes.actions}>
<Tooltip title="Show the material of this entry in the NOMAD Encyclopedia.">
<Button color="primary" href={`${appBase}/encyclopedia/#/material/${data.encyclopedia.material.material_id}`}>
material
</Button>
</Tooltip>
</div>
}
</div>
}
......
......@@ -72,6 +72,11 @@ export const domains = ({
tooltip: 'Aggregates the number of simulated system geometries in all entries.',
renderResultString: count => (<span> that simulate <b>{count.toLocaleString()}</b> unique geometrie{count === 1 ? '' : 's'}</span>)
},
'encyclopedia.material.materials': {
label: 'Materials',
tooltip: 'Shows statistics in terms of materials.',
renderResultString: count => (<span> of <b>{count.toLocaleString()}</b> material{count === 1 ? '' : 's'}</span>)
},
datasets: {
label: 'Datasets',
tooltip: 'Shows statistics in terms of datasets that entries belong to.',
......@@ -152,7 +157,7 @@ export const domains = ({
* the entry view. Needs to work with props: data (the entry data from the API),
* loading (a bool with api loading status).
*/
searchTabs: ['entries', 'datasets', 'groups', 'uploads']
searchTabs: ['entries', 'materials', 'datasets', 'groups', 'uploads']
},
ems: {
name: 'EMS',
......
import React from 'react'
import PropTypes from 'prop-types'
import { makeStyles, TableCell, Toolbar, IconButton, Tooltip } from '@material-ui/core'
import NextIcon from '@material-ui/icons/ChevronRight'
import StartIcon from '@material-ui/icons/SkipPrevious'
import DataTable from '../DataTable'
import DetailsIcon from '@material-ui/icons/MoreHoriz'
import { appBase } from '../../config'
const useStyles = makeStyles(theme => ({
root: {
overflow: 'auto',
paddingLeft: theme.spacing(2),
paddingRight: theme.spacing(2)
},
scrollCell: {
padding: 0
},
scrollBar: {
minHeight: 56,
padding: 0
},
scrollSpacer: {
flexGrow: 1
},
clickableRow: {
cursor: 'pointer'
}
}))
const columns = {
formula: {
label: 'Formula',
render: entry => entry.encyclopedia.material.formula
},
material_name: {
label: 'Name',
render: entry => entry.encyclopedia.material.material_name
},
material_type: {
label: 'Type',
render: entry => entry.encyclopedia.material.material_type
},
spacegroup: {
label: 'Spacegroup',
render: entry => {
const bulk = entry.encyclopedia.material.bulk
return (bulk && bulk.space_group_international_short_symbol) || '-'
}
},
calculations: {
label: 'No calculations',
render: entry => entry.total
}
}
export default function MaterialsList(props) {
const { data, total, materials_after, per_page, onChange, actions } = props
const classes = useStyles()
const materials = data['encyclopedia.material.materials_grouped'] || {values: []}
const results = Object.keys(materials.values).map(id => {
return {
id: id,
total: materials.values[id].total,
...materials.values[id].examples[0]
}
})
const after = materials.after
const perPage = per_page || 10
let paginationText
if (materials_after) {
paginationText = `next ${results.length.toLocaleString()} of ${(total || 0).toLocaleString()}`
} else {
paginationText = `1-${results.length.toLocaleString()} of ${(total || 0).toLocaleString()}`
}
const pagination = <TableCell colSpan={1000} classes={{root: classes.scrollCell}}>
<Toolbar className={classes.scrollBar}>
<span className={classes.scrollSpacer}>&nbsp;</span>
<span>{paginationText}</span>
<IconButton disabled={!materials_after} onClick={() => onChange({materials_grouped_after: null})}>
<StartIcon />
</IconButton>
<IconButton disabled={results.length < perPage} onClick={() => onChange({materials_grouped_after: after})}>
<NextIcon />
</IconButton>
</Toolbar>
</TableCell>
const entryActions = entry => <Tooltip title="Open this material in the Encyclopedia.">
<IconButton href={`${appBase}/encyclopedia/#/material/${entry.encyclopedia.material.material_id}`}>
<DetailsIcon />
</IconButton>
</Tooltip>
return <DataTable
entityLabels={['material', 'materials']}
id={row => row.id}
total={total}
columns={columns}
selectedColumns={['formula', 'material_name', 'material_type', 'spacegroup', 'calculations']}
selectedColumnsKey="materials"
data={results}
rows={perPage}
actions={actions}
pagination={pagination}
entryActions={entryActions}
/>
}
MaterialsList.propTypes = ({
data: PropTypes.object,
total: PropTypes.number,
onChange: PropTypes.func.isRequired,
materials_after: PropTypes.string,
per_page: PropTypes.number,
actions: PropTypes.element
})
......@@ -19,6 +19,7 @@ import UploadsHistogram from './UploadsHistogram'
import QuantityHistogram from './QuantityHistogram'
import SearchContext