Commit fe810c8c authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.10.0' into reprocess

parents 77a9c959 6a69a801
......@@ -47,7 +47,7 @@ python linting:
script:
- cd /app
- python -m pycodestyle --ignore=E501,E701,E731 nomad tests
- python -m pylint --load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin nomad tests
- python -m pylint --load-plugins=pylint_mongoengine,nomad.metainfo.pylint_plugin nomad tests
- python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
except:
refs:
......@@ -176,13 +176,27 @@ deploy prod beta:
- echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- helm dependency update ops/helm/nomad
- helm upgrade --install nomad-prod-test ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- helm upgrade --install nomad-beta ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/beta/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
except:
- /^dev-.*$/
when: manual
deploy prod test:
stage: release
before_script:
- mkdir -p /etc/deploy
- echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- helm dependency update ops/helm/nomad
- helm upgrade --install nomad-test ops/helm/nomad -f ops/helm/nomad/deployments/prod-test-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/test/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
except:
- /^dev-.*$/
when: manual
release latest image:
stage: release
script:
......
Subproject commit c234b1b184534bbe334acfcd59fc4e0618612814
Subproject commit 52e4caf2dab39e1ddf3fbff7d9c4adef06851f87
'''
The scenario is that you have a lot of <name>-*.zip files for upload. This script
will add a nomad.json to the uploads, upload them 1 at a time, watch the processing, repeat.
'''
from typing import Dict, Any
from bravado.requests_client import RequestsClient, Authenticator
from bravado.client import SwaggerClient
from keycloak import KeycloakOpenID
from urllib.parse import urlparse
import time
import os.path
import sys
import zipfile
from nomad.client import KeycloakAuthenticator
from nomad import config
nomad_url = 'http://nomad-lab.eu/prod/rae/api'
user = 'youruser'
password = 'yourpassword'
uploader_id = None
# create the bravado client
http_client = RequestsClient()
http_client.authenticator = KeycloakAuthenticator(
host=urlparse(nomad_url).netloc,
user=user,
password=password,
server_url=config.keycloak.server_url,
realm_name=config.keycloak.realm_name,
client_id=config.keycloak.client_id)
client = SwaggerClient.from_url('%s/swagger.json' % nomad_url, http_client=http_client)
def upload(
path: str, local_path: bool = False, metadata_path: str = None,
publish_directly: bool = False, uploader_id: str = None):
'''
Arguments:
path: The file path to the upload file.
local: If this file should be uploaded with &local_path
metadata_path: Optional nomad.(yaml|json) metadata file
publish_directly: If the upload should be published directly
'''
assert os.path.isfile(path), f'The {path} is not a file'
# add metadata
if metadata_path is not None:
assert os.path.isfile(metadata_path), f'The {metadata_path} is not a file'
assert os.path.basename(metadata_path).endswith('.json'), f'The {metadata_path} is not a nomad metadata file'
assert path.endswith('.zip'), 'Adding nomad metadata is only supported for .zip files'
with zipfile.ZipFile(path, 'a') as zip:
zip.write(metadata_path, 'nomad.json')
# upload
print(f'uploading {path}')
kwargs: Dict[str, Any] = {}
if publish_directly:
kwargs['publish_directly'] = True
if uploader_id is not None:
kwargs['uploader_id'] = uploader_id
if local_path:
upload = client.uploads.upload(local_path=path, **kwargs).response().result
else:
with open(path, 'rb') as f:
upload = client.uploads.upload(file=f, **kwargs).response().result
print(f'processing {path}')
while upload.tasks_running:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
time.sleep(3)
print('processed: %d, failures: %d' % (upload.processed_calcs, upload.failed_calcs))
# check if processing was a success
if upload.tasks_status != 'SUCCESS':
print('something went wrong')
print('errors: %s' % str(upload.errors))
# try to delete the unsuccessful upload
client.uploads.delete_upload(upload_id=upload.upload_id).response().result
return False
return True
if __name__ == '__main__':
metadata_path = None
if sys.argv[1].endswith('json'):
metadata_path = sys.argv[1]
paths = sys.argv[2:]
else:
paths = sys.argv[1:]
for path in paths:
upload(
path, metadata_path=metadata_path, local_path=True, publish_directly=True,
uploader_id=uploader_id)
......@@ -23,6 +23,7 @@ import ElectronicStructureOverview from '../visualization/ElectronicStructureOve
import VibrationalOverview from '../visualization/VibrationalOverview'
import { ApiDialog } from '../ApiDialogButton'
import { Structure } from '../visualization/Structure'
import NoData from '../visualization/NoData'
import Actions from '../Actions'
import Quantity from '../Quantity'
import { RecoilRoot } from 'recoil'
......@@ -169,32 +170,40 @@ export default function DFTEntryOverview({data}) {
const section_run = archive.section_run[0]
let section_method = null
const sccs = section_run.section_single_configuration_calculation
for (let i = sccs.length - 1; i > -1; --i) {
const scc = sccs[i]
if (!e_dos && scc.section_dos) {
if (scc.section_dos[scc.section_dos.length - 1].dos_kind !== 'vibrational') {
e_dos = {
'section_system': scc.single_configuration_calculation_to_system_ref,
'section_method': scc.single_configuration_calculation_to_system_ref,
'section_dos': scc.section_dos[scc.section_dos.length - 1]
if (sccs) {
for (let i = sccs.length - 1; i > -1; --i) {
const scc = sccs[i]
if (!e_dos && scc.section_dos) {
const first_dos = scc.section_dos[scc.section_dos.length - 1]
if (!_.isEmpty(first_dos)) {
if (first_dos.dos_kind !== 'vibrational') {
e_dos = {
'section_system': scc.single_configuration_calculation_to_system_ref,
'section_method': scc.single_configuration_calculation_to_system_ref,
'section_dos': scc.section_dos[scc.section_dos.length - 1]
}
}
}
}
}
if (!e_bs && scc.section_k_band) {
if (scc.section_k_band[scc.section_k_band.length - 1].band_structure_kind !== 'vibrational') {
e_bs = {
'section_system': scc.single_configuration_calculation_to_system_ref,
'section_method': scc.single_configuration_calculation_to_system_ref,
'section_k_band': scc.section_k_band[scc.section_k_band.length - 1]
if (!e_bs && scc.section_k_band) {
const first_band = scc.section_k_band[scc.section_k_band.length - 1]
if (!_.isEmpty(first_band)) {
if (first_band.band_structure_kind !== 'vibrational') {
e_bs = {
'section_system': scc.single_configuration_calculation_to_system_ref,
'section_method': scc.single_configuration_calculation_to_system_ref,
'section_k_band': scc.section_k_band[scc.section_k_band.length - 1]
}
}
}
}
}
if (section_method !== false) {
let iMethod = scc.single_configuration_to_calculation_method_ref
if (section_method === null) {
section_method = iMethod
} else if (iMethod !== section_method) {
section_method = false
if (section_method !== false) {
let iMethod = scc.single_configuration_to_calculation_method_ref
if (section_method === null) {
section_method = iMethod
} else if (iMethod !== section_method) {
section_method = false
}
}
}
}
......@@ -295,9 +304,17 @@ export default function DFTEntryOverview({data}) {
}
// Get method details. Any referenced core_setttings will also be taken
// into account
// into account. If there were no SCCs from which the method could be
// selected, simply select the last available method.
if (section_method) {
section_method = resolveRef(section_method, archive)
} else {
const methods = section_run.section_method
if (methods) {
section_method = methods[methods.length - 1]
}
}
if (section_method) {
const refs = section_method?.section_method_to_method_refs
if (refs) {
for (const ref of refs) {
......@@ -321,17 +338,19 @@ export default function DFTEntryOverview({data}) {
// Get the representative system by looping over systems
let reprSys = null
const systems = section_run.section_system
for (let i = systems.length - 1; i > -1; --i) {
const sys = systems[i]
if (!reprSys && sys.is_representative) {
const reprSys = {
'species': sys.atom_species,
'cell': sys.lattice_vectors ? convertSI(sys.lattice_vectors, 'meter', {length: 'angstrom'}, false) : undefined,
'positions': convertSI(sys.atom_positions, 'meter', {length: 'angstrom'}, false),
'pbc': sys.configuration_periodic_dimensions
if (systems) {
for (let i = systems.length - 1; i > -1; --i) {
const sys = systems[i]
if (!reprSys && sys.is_representative) {
const reprSys = {
'species': sys.atom_species,
'cell': sys.lattice_vectors ? convertSI(sys.lattice_vectors, 'meter', {length: 'angstrom'}, false) : undefined,
'positions': convertSI(sys.atom_positions, 'meter', {length: 'angstrom'}, false),
'pbc': sys.configuration_periodic_dimensions
}
structs.original = reprSys
break
}
structs.original = reprSys
break
}
}
......@@ -363,13 +382,7 @@ export default function DFTEntryOverview({data}) {
// Figure out which actions are available for this entry
const actions = useMemo(() => {
const buttons = [
{
tooltip: 'Show the API access code',
onClick: (event) => { setShowAPIDialog(!showAPIDialog) },
content: 'API access'
}
]
const buttons = []
if (encyclopediaEnabled && data?.encyclopedia?.material?.material_id) {
buttons.push(
{
......@@ -379,6 +392,13 @@ export default function DFTEntryOverview({data}) {
}
)
}
buttons.push(
{
tooltip: 'Show the API access code',
onClick: (event) => { setShowAPIDialog(!showAPIDialog) },
content: 'API access'
}
)
return buttons
}, [data, showAPIDialog])
......@@ -489,7 +509,10 @@ export default function DFTEntryOverview({data}) {
</Box>
</Grid>
<Grid item xs={7} style={{marginTop: '-2rem'}}>
<Structure systems={structures} aspectRatio={1.5} />
{(loading || !_.isEmpty(structures))
? <Structure systems={structures} materialType={data?.dft?.system} aspectRatio={1.5}/>
: <NoData aspectRatio={1.5}/>
}
</Grid>
</Grid>
</PropertyCard>
......
......@@ -104,12 +104,14 @@ export class EntryListUnstyled extends React.Component {
overflow: 'auto'
},
entryDetails: {
paddingTop: theme.spacing(3),
paddingLeft: theme.spacing(3),
paddingRight: theme.spacing(3)
paddingTop: theme.spacing(2),
paddingLeft: theme.spacing(2),
paddingRight: theme.spacing(2)
},
entryDetailsContents: {
display: 'flex'
display: 'flex',
maxWidth: 1024,
margin: 'auto'
},
entryDetailsRow: {
paddingRight: theme.spacing(3)
......@@ -121,9 +123,7 @@ export class EntryListUnstyled extends React.Component {
flexShrink: 0,
justifyContent: 'flex-end',
marginBottom: theme.spacing(1),
marginLeft: theme.spacing(0.5),
marginRight: theme.spacing(0.5),
marginTop: theme.spacing(1)
marginTop: theme.spacing(2)
}
})
......@@ -247,17 +247,6 @@ export class EntryListUnstyled extends React.Component {
}
}
selectionQuery() {
const { selected } = this.state
if (selected) {
return {
'calc_id': selected.join(',')
}
} else {
return this.props.query
}
}
renderEntryDetails(row) {
const { classes } = this.props
const domain = (row.domain && domains[row.domain]) || domains.dft
......@@ -375,7 +364,7 @@ export class EntryListUnstyled extends React.Component {
/>
const example = selected && selected.length > 0 ? results.find(d => d.calc_id === selected[0]) : results[0]
const selectQuery = (selected && selected.length > 0) ? {calc_id: selected} : query
const selectQuery = (selected && selected.length > 0) ? {calc_id: selected, owner: query['owner']} : query
const createActions = (props, moreActions) => <React.Fragment>
{example && editable ? <EditUserMetadataDialog
example={example} total={selected === null ? totalNumber : selected.length}
......
......@@ -27,7 +27,7 @@ import Plot from '../visualization/Plot'
import { convertSI, distance, mergeObjects } from '../../utils'
import { withErrorHandler } from '../ErrorHandler'
function BandStructure({data, layout, aspectRatio, className, classes, onRelayout, onAfterPlot, onRedraw, onRelayouting, onReset, unitsState}) {
function BandStructure({data, layout, aspectRatio, className, classes, unitsState, ...other}) {
const [finalData, setFinalData] = useState(undefined)
const [pathSegments, setPathSegments] = useState(undefined)
const units = useRecoilValue(unitsState)
......@@ -246,11 +246,7 @@ function BandStructure({data, layout, aspectRatio, className, classes, onRelayou
layout={finalLayout}
aspectRatio={aspectRatio}
floatTitle={'Band structure'}
onRelayout={onRelayout}
onAfterPlot={onAfterPlot}
onRedraw={onRedraw}
onRelayouting={onRelayouting}
onReset={onReset}
{...other}
>
</Plot>
</Box>
......@@ -263,11 +259,6 @@ BandStructure.propTypes = {
aspectRatio: PropTypes.number,
classes: PropTypes.object,
className: PropTypes.string,
onAfterPlot: PropTypes.func,
onRedraw: PropTypes.func,
onRelayout: PropTypes.func,
onRelayouting: PropTypes.func,
onReset: PropTypes.func,
unitsState: PropTypes.object // Recoil atom containing the unit configuration
}
......
......@@ -27,7 +27,7 @@ import Plot from '../visualization/Plot'
import { convertSI, convertSILabel, mergeObjects } from '../../utils'
import { withErrorHandler } from '../ErrorHandler'
function DOS({data, layout, resetLayout, aspectRatio, className, classes, onRelayout, onAfterPlot, onRedraw, onRelayouting, onReset, unitsState}) {
function DOS({data, layout, resetLayout, aspectRatio, className, classes, unitsState, ...other}) {
const [finalData, setFinalData] = useState(undefined)
const units = useRecoilValue(unitsState)
......@@ -128,11 +128,7 @@ function DOS({data, layout, resetLayout, aspectRatio, className, classes, onRela
resetLayout={resetLayout}
aspectRatio={aspectRatio}
floatTitle="Density of states"
onRelayout={onRelayout}
onAfterPlot={onAfterPlot}
onRedraw={onRedraw}
onRelayouting={onRelayouting}
onReset={onReset}
{...other}
>
</Plot>
</Box>
......@@ -146,11 +142,6 @@ DOS.propTypes = {
aspectRatio: PropTypes.number,
classes: PropTypes.object,
className: PropTypes.string,
onAfterPlot: PropTypes.func,
onRedraw: PropTypes.func,
onRelayout: PropTypes.func,
onRelayouting: PropTypes.func,
onReset: PropTypes.func,
unitsState: PropTypes.object // Recoil atom containing the unit configuration
}
......
......@@ -15,13 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import React, { useState, useCallback } from 'react'
import React, { useState, useCallback, useMemo } from 'react'
import { Subject } from 'rxjs'
import PropTypes from 'prop-types'
import {
Box,
Typography
} from '@material-ui/core'
import DOS from './DOS'
import NoData from './NoData'
import BandStructure from './BandStructure'
import BrillouinZone from './BrillouinZone'
import { RecoilRoot } from 'recoil'
......@@ -38,6 +40,10 @@ function ElectronicStructureOverview({data, range, className, classes, raiseErro
yaxis: {range: range}
})
// RxJS subject for efficiently propagating y axis changes between DOS and BS
const bsYSubject = useMemo(() => new Subject(), [])
const dosYSubject = useMemo(() => new Subject(), [])
// Styles
const useStyles = makeStyles((theme) => {
return {
......@@ -45,7 +51,7 @@ function ElectronicStructureOverview({data, range, className, classes, raiseErro
display: 'flex',
flexDirection: 'row',
justifyContent: 'flex-start',
alignItems: 'center',
alignItems: 'flex-start',
width: '100%',
height: '100%',
flexWrap: 'wrap'
......@@ -66,58 +72,50 @@ function ElectronicStructureOverview({data, range, className, classes, raiseErro
// Synchronize panning between BS/DOS plots
const handleBSRelayouting = useCallback((event) => {
if (data.dos) {
let update = {
yaxis: {
autorange: false,
range: [event['yaxis.range[0]'], event['yaxis.range[1]']]
}
}
setDosLayout(update)
let update = {yaxis: {range: [event['yaxis.range[0]'], event['yaxis.range[1]']]}}
bsYSubject.next(update)
}
}, [data])
}, [data, bsYSubject])
const handleDOSRelayouting = useCallback((event) => {
if (data.bs) {
let update = {
yaxis: {
autorange: false,
range: [event['yaxis.range[0]'], event['yaxis.range[1]']]
}
}
setBsLayout(update)
let update = {yaxis: {range: [event['yaxis.range[0]'], event['yaxis.range[1]']]}}
dosYSubject.next(update)
}
}, [data])
}, [data, dosYSubject])
return (
<RecoilRoot>
<Box className={style.row}>
{data.bs
? <Box className={style.bs}>
<Typography variant="subtitle1" align='center'>Band structure</Typography>
<BandStructure
<Box className={style.bs}>
<Typography variant="subtitle1" align='center'>Band structure</Typography>
{data.bs
? <BandStructure
data={data?.bs?.section_k_band}
layout={bsLayout}
aspectRatio={1.2}
unitsState={unitsState}
onRelayouting={handleBSRelayouting}
onReset={() => { setDosLayout({yaxis: {range: range}}) }}
layoutSubject={dosYSubject}
></BandStructure>
</Box>
: null
}
{data.dos
? <Box className={style.dos}>
<Typography variant="subtitle1" align='center'>Density of states</Typography>
<DOS
: <NoData aspectRatio={1.2}/>
}
</Box>
<Box className={style.dos}>
<Typography variant="subtitle1" align='center'>Density of states</Typography>
{data.dos
? <DOS
data={data.dos.section_dos}
layout={dosLayout}
aspectRatio={0.6}
onRelayouting={handleDOSRelayouting}
onReset={() => { setBsLayout({yaxis: {range: range}}) }}
unitsState={unitsState}
layoutSubject={bsYSubject}
></DOS>
</Box>
: null
}
: <NoData aspectRatio={0.6}/>
}
</Box>
{data.bs
? <Box className={style.bz}>
<Typography variant="subtitle1" align='center'>Brillouin zone</Typography>
......
......@@ -15,7 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import React, { useState, useCallback, useMemo } from 'react'
import React, { useCallback, useMemo } from 'react'
import { Subject } from 'rxjs'
import PropTypes from 'prop-types'
import {
Box,
......@@ -28,7 +29,8 @@ import { Structure } from '../visualization/Structure'
import { ErrorHandler, withErrorHandler } from '../ErrorHandler'
function GeoOptOverview({data, className, classes}) {
const [step, setStep] = useState(0)
// RxJS subject for efficiently propagating changes in structure information
const positionsSubject = useMemo(() => new Subject(), [])
// Styles
const useStyles = makeStyles((theme) => {
......@@ -129,8 +131,8 @@ function GeoOptOverview({data, className, classes}) {
// Handles hover event on the plot to update the currently shown structure
const handleHover = useCallback((event) => {
setStep(event.points[0].x)
}, [])
positionsSubject.next(data.structures[event.points[0].x].positions)
}, [data, positionsSubject])
return (
<Box className={style.root}>
......@@ -150,10 +152,10 @@ function GeoOptOverview({data, className, classes}) {
<Box className={style.structure}>
<Typography variant="subtitle1" align='center'>Optimization trajectory</Typography>
<Structure
system={data.structures[step]}