diff --git a/gui/src/components/DefinitionTitle.js b/gui/src/components/DefinitionTitle.js
new file mode 100644
index 0000000000000000000000000000000000000000..9501ab9377e5c7d78462cb53e9d8d6829ea24093
--- /dev/null
+++ b/gui/src/components/DefinitionTitle.js
@@ -0,0 +1,106 @@
+/*
+ * Copyright The NOMAD Authors.
+ *
+ * This file is part of NOMAD. See https://nomad-lab.eu for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React from 'react'
+import { makeStyles } from '@material-ui/core/styles'
+import { Typography, Tooltip } from '@material-ui/core'
+import PropTypes from 'prop-types'
+import clsx from 'clsx'
+import Ellipsis from './visualization/Ellipsis'
+
+/**
+ * Simple component for displaying titles and corresponding tooltips for
+ * metainfo definitions.
+ */
+const useStaticStyles = makeStyles(theme => ({
+ root: {
+ },
+ text: {
+ },
+ subtitle2: {
+ color: theme.palette.grey[800]
+ },
+ right: {
+ overflow: 'hidden'
+ },
+ down: {
+ overflow: 'hidden',
+ writingMode: 'vertical-rl',
+ textOrientation: 'mixed'
+ },
+ up: {
+ overflow: 'hidden',
+ writingMode: 'vertical-rl',
+ textOrientation: 'mixed',
+ transform: 'rotate(-180deg)'
+ }
+}))
+export const DefinitionTitle = React.memo(({
+ label,
+ description,
+ variant,
+ TooltipProps,
+ onMouseDown,
+ onMouseUp,
+ className,
+ classes,
+ rotation,
+ section,
+ noWrap
+}) => {
+ const styles = useStaticStyles({classes})
+
+ return <Tooltip title={description || ''} interactive enterDelay={400} enterNextDelay={400} {...(TooltipProps || {})}>
+ <div className={clsx(className, styles.root,
+ rotation === 'right' && styles.right,
+ rotation === 'down' && styles.down,
+ rotation === 'up' && styles.up
+ )}>
+ <Typography
+ noWrap={noWrap}
+ className={clsx(styles.text, (!section) && (variant === "subtitle2") && styles.subtitle2)}
+ variant={variant}
+ onMouseDown={onMouseDown}
+ onMouseUp={onMouseUp}
+ >
+ <Ellipsis>{label}</Ellipsis>
+ </Typography>
+ </div>
+ </Tooltip>
+})
+
+DefinitionTitle.propTypes = {
+ quantity: PropTypes.string,
+ label: PropTypes.string,
+ description: PropTypes.oneOfType([PropTypes.string, PropTypes.node]),
+ variant: PropTypes.string,
+ className: PropTypes.string,
+ classes: PropTypes.object,
+ rotation: PropTypes.oneOf(['up', 'right', 'down']),
+ TooltipProps: PropTypes.object, // Properties forwarded to the Tooltip
+ onMouseDown: PropTypes.func,
+ onMouseUp: PropTypes.func,
+ placement: PropTypes.string,
+ noWrap: PropTypes.bool,
+ section: PropTypes.string
+}
+
+DefinitionTitle.defaultProps = {
+ variant: 'body2',
+ rotation: 'right',
+ noWrap: true
+}
diff --git a/gui/src/components/plotting/PlotHistogram.js b/gui/src/components/plotting/PlotHistogram.js
index d83861097ff6ccdf9e8119e80f53be470175fac8..d984bc0039012d292af7ff7eb5791266933953de 100644
--- a/gui/src/components/plotting/PlotHistogram.js
+++ b/gui/src/components/plotting/PlotHistogram.js
@@ -29,11 +29,11 @@ import { InputTextField } from '../search/input/InputText'
import Placeholder from '../visualization/Placeholder'
import PlotAxis from './PlotAxis'
import PlotBar from './PlotBar'
-import FilterTitle from '../search/FilterTitle'
import { guiState } from '../GUIMenu'
import PropTypes from 'prop-types'
import { getScaler } from './common'
import { dateFormat } from '../../config'
+import { DefinitionTitle } from '../DefinitionTitle'
/**
* An interactive histogram for numeric values.
@@ -210,7 +210,6 @@ const PlotHistogram = React.memo(({
}
})
const dynamicStyles = useDynamicStyles()
-
const aggIndicator = useRecoilValue(guiState('aggIndicator'))
const oldRangeRef = useRef()
const artificialRange = 1
@@ -534,12 +533,11 @@ const PlotHistogram = React.memo(({
}
const titleComp = <div className={styles.title}>
- <FilterTitle
+ <DefinitionTitle
variant="subtitle2"
classes={titleClasses}
- quantity={xAxis.quantity}
label={xAxis.title}
- unit={xAxis.unit}
+ description={xAxis.description}
noWrap={false}
/>
</div>
diff --git a/gui/src/components/plotting/PlotScatter.js b/gui/src/components/plotting/PlotScatter.js
index d856bf9e2f2569068f8d54857fd1a2dc28dc5821..34b91b917b430228cb7c978930275b3933e7d63e 100644
--- a/gui/src/components/plotting/PlotScatter.js
+++ b/gui/src/components/plotting/PlotScatter.js
@@ -18,13 +18,25 @@
import React, {useState, useEffect, useMemo, useCallback, forwardRef} from 'react'
import PropTypes from 'prop-types'
import { makeStyles, useTheme } from '@material-ui/core'
-import { hasWebGLSupport } from '../../utils'
+import { hasWebGLSupport, DType } from '../../utils'
import * as d3 from 'd3'
-import FilterTitle from '../search/FilterTitle'
+import { DefinitionTitle } from '../DefinitionTitle'
import Plot from './Plot'
import { useHistory } from 'react-router-dom'
import { getUrl } from '../nav/Routes'
+function getAxisType(type, scale) {
+ return type === DType.Timestamp && scale === 'linear'
+ ? 'date'
+ : scale
+}
+
+function transformData(type, data) {
+ return type === DType.Timestamp
+ ? data.map((iso) => new Date(iso).getTime())
+ : data
+}
+
/**
* A Plotly-based interactive scatter plot.
*/
@@ -86,7 +98,9 @@ const useStyles = makeStyles(theme => ({
axisTitle: {
fontSize: '0.75rem'
}
+
}))
+
const PlotScatter = React.memo(forwardRef((
{
data,
@@ -103,8 +117,8 @@ const PlotScatter = React.memo(forwardRef((
'data-testid': testID
}, canvas) => {
const styles = useStyles()
- const theme = useTheme()
const titleClasses = {text: styles.axisTitle}
+ const theme = useTheme()
const [finalData, setFinalData] = useState(!data ? data : undefined)
const history = useHistory()
@@ -118,6 +132,12 @@ const PlotScatter = React.memo(forwardRef((
return
}
+ // Map the data depending on axis types. This manual transformation is
+ // needed because the plotly automatic axis type detection does not work
+ // when scaling option is read from the axis configuration.
+ data.x = transformData(xAxis.dtype, data.x)
+ data.y = transformData(yAxis.dtype, data.y)
+
const hoverTemplate = (xLabel, yLabel, colorLabel, xUnit, yUnit, colorUnit) => {
let template = `<b>Click to go to entry page</b>` +
`<br>` +
@@ -251,7 +271,7 @@ const PlotScatter = React.memo(forwardRef((
})
}
setFinalData(traces)
- }, [colorAxis?.search_quantity, colorAxis?.title, colorAxis?.unit, data, discrete, theme, xAxis.title, xAxis.unit, yAxis.title, yAxis.unit])
+ }, [colorAxis?.search_quantity, colorAxis?.title, colorAxis?.unit, data, discrete, theme, xAxis.dtype, xAxis.title, xAxis.unit, yAxis.dtype, yAxis.title, yAxis.unit])
const layout = useMemo(() => {
return {
@@ -272,12 +292,12 @@ const PlotScatter = React.memo(forwardRef((
y: 1
},
xaxis: {
- type: xAxis.scale,
+ type: getAxisType(xAxis.dtype, xAxis.scale),
fixedrange: false,
autorange: autorange
},
yaxis: {
- type: yAxis.scale,
+ type: getAxisType(yAxis.dtype, yAxis.scale),
fixedrange: false,
autorange: autorange
},
@@ -295,7 +315,7 @@ const PlotScatter = React.memo(forwardRef((
// both. This is a general problem in trying to 'reactify' a non-react library
// like Plotly.
// eslint-disable-next-line react-hooks/exhaustive-deps
- }, [autorange, xAxis.scale, yAxis.scale])
+ }, [autorange, xAxis.dtype, xAxis.scale, yAxis.dtype, yAxis.scale])
// Change dragmode
useEffect(() => {
@@ -317,13 +337,12 @@ const PlotScatter = React.memo(forwardRef((
return <div className={styles.root}>
<div className={styles.yaxis}>
- <FilterTitle
- variant="subtitle2"
- classes={titleClasses}
- quantity={yAxis.quantity}
+ <DefinitionTitle
label={yAxis.title}
- unit={yAxis.unit}
+ description={yAxis.description}
+ variant="subtitle2"
rotation="up"
+ classes={titleClasses}
/>
</div>
<div className={styles.plot}>
@@ -344,24 +363,21 @@ const PlotScatter = React.memo(forwardRef((
</div>
<div className={styles.square} />
<div className={styles.xaxis}>
- <FilterTitle
+ <DefinitionTitle
+ label={xAxis.title}
+ description={xAxis.description}
variant="subtitle2"
classes={titleClasses}
- quantity={xAxis.quantity}
- label={xAxis.title}
- unit={xAxis.unit}
/>
</div>
{!discrete && colorAxis &&
<div className={styles.color}>
- <FilterTitle
+ <DefinitionTitle
+ label={colorAxis.title}
+ description={colorAxis.description}
+ rotation="down"
variant="subtitle2"
classes={titleClasses}
- rotation="down"
- quantity={colorAxis.quantity}
- unit={colorAxis.unit}
- label={colorAxis.title}
- description=""
/>
</div>
}
diff --git a/gui/src/components/plotting/PlotScatter.spec.js b/gui/src/components/plotting/PlotScatter.spec.js
new file mode 100644
index 0000000000000000000000000000000000000000..95d43868cb00a8d104b330aa21bce48d6c988030
--- /dev/null
+++ b/gui/src/components/plotting/PlotScatter.spec.js
@@ -0,0 +1,52 @@
+/*
+ * Copyright The NOMAD Authors.
+ *
+ * This file is part of NOMAD. See https://nomad-lab.eu for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React, { useRef } from 'react'
+import { renderNoAPI, screen } from '../conftest.spec'
+import PlotScatter from './PlotScatter'
+import { DType } from '../../utils'
+
+const ScatterPlotWrapper = React.memo((props) => {
+ const canvas = useRef()
+ return <PlotScatter {...props} ref={canvas}/>
+})
+
+test.each([
+ [
+ 'linear timestamp',
+ {x: ["2014-12-01T00:00:00+00:00"], y: ["2015-12-01T00:00:00+00:00"]},
+ {title: 'Test', dtype: DType.Timestamp, scale: 'linear'},
+ ['Dec 1, 2014', 'Dec 1, 2015']
+ ],
+ [
+ 'log float',
+ {x: [1, 10], y: [100, 1000]},
+ {title: 'Test', dtype: DType.Float, scale: 'log'},
+ ['1', '10', '100', '1000']
+ ]
+])('%s', async (id, data, axis, ticks) => {
+ renderNoAPI(<ScatterPlotWrapper
+ data={data}
+ xAxis={axis}
+ yAxis={axis}
+ />)
+
+ // Check that the correct plot ticks are found
+ for (const tick of ticks) {
+ expect(await screen.findByText(tick)).toBeInTheDocument()
+ }
+})
diff --git a/gui/src/components/plotting/common.js b/gui/src/components/plotting/common.js
index 559fc3f8599d23565126b8e78cae0f11fb0c7941..1dd7a4a179469f9c8f5d353932d04d7a3cbb6f7f 100644
--- a/gui/src/components/plotting/common.js
+++ b/gui/src/components/plotting/common.js
@@ -15,7 +15,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+import React from 'react'
import { range, size } from 'lodash'
+import { Typography } from '@material-ui/core'
import { scalePow, scaleLog } from 'd3-scale'
import {
format,
@@ -497,17 +499,41 @@ export function getPlotTracesVertical(plots, theme) {
export function getAxisConfig(axis, filterData, units) {
const {quantity} = parseJMESPath(axis?.search_quantity)
const filter = filterData[quantity]
- const title = axis.title || filter?.label || getDisplayLabel(filter)
const dtype = filter?.dtype
const unit = axis.unit
? new Unit(axis.unit)
: new Unit(filter?.unit || 'dimensionless').toSystem(units)
+ // Create the final label
+ let title = axis.title || filter?.label || getDisplayLabel(filter)
+ let finalUnit
+ if (unit) {
+ finalUnit = new Unit(unit).label()
+ } else if (filter?.unit) {
+ finalUnit = new Unit(filter.unit).toSystem(units).label()
+ }
+ if (finalUnit) {
+ title = `${title} (${finalUnit})`
+ }
+
+ // Determine the final description
+ let description = axis.description || filter?.description || ''
+ if (description && quantity) {
+ description = (
+ <>
+ <Typography>{title}</Typography>
+ <b>Description: </b>{description}<br/>
+ <b>Path: </b>{quantity}
+ </>
+ )
+ }
+
return {
...axis,
+ description,
title,
unit,
dtype,
- quantity: quantity
+ quantity
}
}
diff --git a/gui/src/components/search/FilterTitle.js b/gui/src/components/search/FilterTitle.js
index 941a3537eba1176cff1f280492255afca5140b3e..b31834863d7cf7284bd49822a808d19d26735367 100644
--- a/gui/src/components/search/FilterTitle.js
+++ b/gui/src/components/search/FilterTitle.js
@@ -16,61 +16,28 @@
* limitations under the License.
*/
import React, { useMemo, useContext } from 'react'
-import { makeStyles } from '@material-ui/core/styles'
-import { Typography, Tooltip } from '@material-ui/core'
+import { Typography } from '@material-ui/core'
import PropTypes from 'prop-types'
-import clsx from 'clsx'
import { useSearchContext } from './SearchContext'
import { inputSectionContext } from './input/InputNestedObject'
import { Unit } from '../units/Unit'
import { useUnitContext } from '../units/UnitContext'
-import Ellipsis from '../visualization/Ellipsis'
+import { DefinitionTitle } from '../DefinitionTitle'
/**
- * Title for a metainfo quantity or section that is used in a search context.
- * By default the label, description and unit are automatically retrieved from
- * the filter config.
+ * Title for a metainfo quantity or section that is used inside a search
+ * context. By default the label, description and unit are automatically
+ * retrieved from the filter config.
*/
-const useStaticStyles = makeStyles(theme => ({
- root: {
- },
- text: {
- },
- subtitle2: {
- color: theme.palette.grey[800]
- },
- right: {
- overflow: 'hidden'
- },
- down: {
- overflow: 'hidden',
- writingMode: 'vertical-rl',
- textOrientation: 'mixed'
- },
- up: {
- overflow: 'hidden',
- writingMode: 'vertical-rl',
- textOrientation: 'mixed',
- transform: 'rotate(-180deg)'
- }
-}))
const FilterTitle = React.memo(({
quantity,
label,
description,
unit,
- variant,
- TooltipProps,
- onMouseDown,
- onMouseUp,
- className,
- classes,
- rotation,
disableUnit,
- noWrap
+ ...rest
}) => {
- const styles = useStaticStyles({classes})
- const { filterData } = useSearchContext()
+ const {filterData} = useSearchContext()
const sectionContext = useContext(inputSectionContext)
const {units} = useUnitContext()
const section = sectionContext?.section
@@ -93,55 +60,31 @@ const FilterTitle = React.memo(({
}, [filterData, quantity, units, label, unit, disableUnit])
// Determine the final description
- const finalDescription = description || (quantity && filterData[quantity]?.description) || ''
- const tooltip = (quantity)
- ? <>
- <Typography>{finalLabel}</Typography>
- <b>Description: </b>{finalDescription || '-'}<br/>
- <b>Path: </b>{quantity}
- </>
- : finalDescription || ''
+ let finalDescription = description || filterData[quantity]?.description || ''
+ if (finalDescription && quantity) {
+ finalDescription = (
+ <>
+ <Typography>{finalLabel}</Typography>
+ <b>Description: </b>{finalDescription}<br/>
+ <b>Path: </b>{quantity}
+ </>
+ )
+ }
- return <Tooltip title={tooltip} interactive enterDelay={400} enterNextDelay={400} {...(TooltipProps || {})}>
- <div className={clsx(className, styles.root,
- rotation === 'right' && styles.right,
- rotation === 'down' && styles.down,
- rotation === 'up' && styles.up
- )}>
- <Typography
- noWrap={noWrap}
- className={clsx(styles.text, (!section) && (variant === "subtitle2") && styles.subtitle2)}
- variant={variant}
- onMouseDown={onMouseDown}
- onMouseUp={onMouseUp}
- >
- <Ellipsis>{finalLabel}</Ellipsis>
- </Typography>
- </div>
- </Tooltip>
+ return <DefinitionTitle
+ label={finalLabel}
+ description={finalDescription}
+ section={section}
+ {...rest}
+ />
})
FilterTitle.propTypes = {
quantity: PropTypes.string,
label: PropTypes.string,
- unit: PropTypes.oneOfType([PropTypes.string, PropTypes.object]),
description: PropTypes.string,
- variant: PropTypes.string,
- className: PropTypes.string,
- classes: PropTypes.object,
- rotation: PropTypes.oneOf(['up', 'right', 'down']),
- disableUnit: PropTypes.bool,
- TooltipProps: PropTypes.object, // Properties forwarded to the Tooltip
- onMouseDown: PropTypes.func,
- onMouseUp: PropTypes.func,
- placement: PropTypes.string,
- noWrap: PropTypes.bool
-}
-
-FilterTitle.defaultProps = {
- variant: 'body2',
- rotation: 'right',
- noWrap: true
+ unit: PropTypes.oneOfType([PropTypes.string, PropTypes.object]),
+ disableUnit: PropTypes.bool
}
export default FilterTitle
diff --git a/gui/src/components/search/Query.js b/gui/src/components/search/Query.js
index c2938193d88bddf254becc822372a6effdefc3bc..972de233c6d389712621db0cd6667f50703489b1 100644
--- a/gui/src/components/search/Query.js
+++ b/gui/src/components/search/Query.js
@@ -236,10 +236,10 @@ const createChips = (name, filterValue, onDelete, filterData, units) => {
}
}
- createRangeChip('lte', '<=')
- createRangeChip('lt', '<')
createRangeChip('gte', '>=')
createRangeChip('gt', '>')
+ createRangeChip('lte', '<=')
+ createRangeChip('lt', '<')
} else {
chips.push({ comp: createChip(serializer(filterValue), () => onDelete(undefined)), op })
}
diff --git a/gui/src/components/search/input/InputHistogram.js b/gui/src/components/search/input/InputHistogram.js
index 9a95c284a5db459dcf06ff28f3d01f8efdfe5124..d90e7e5b71887b378383da2a6fc898813ec1861f 100644
--- a/gui/src/components/search/input/InputHistogram.js
+++ b/gui/src/components/search/input/InputHistogram.js
@@ -246,12 +246,8 @@ export const Histogram = React.memo(({
setPlotData({
xAxis: {
- search_quantity: x.search_quantity,
- quantity: x.quantity,
- unit: x.unit,
+ ...x,
unitStorage: unitStorage,
- dtype: x.dtype,
- title: x.title,
min: minLocal,
max: maxLocal
},
@@ -259,7 +255,7 @@ export const Histogram = React.memo(({
step: stepHistogram,
data: agg.data
})
- }, [loading, nBins, agg, minLocal, maxLocal, stepHistogram, unitStorage, x.search_quantity, x.quantity, x.unit, x.dtype, x.title, x.scale, y])
+ }, [loading, nBins, agg, minLocal, maxLocal, stepHistogram, unitStorage, x, y])
// Function for converting search values into the currently selected unit
// system.
diff --git a/gui/src/components/search/widgets/WidgetScatterPlot.js b/gui/src/components/search/widgets/WidgetScatterPlot.js
index 04fd99367bd566faf55a8b77264ca25c4fba8446..9e57ec90d996b64ae9e6cb19d1a38ccead8e12ed 100644
--- a/gui/src/components/search/widgets/WidgetScatterPlot.js
+++ b/gui/src/components/search/widgets/WidgetScatterPlot.js
@@ -264,10 +264,10 @@ export const WidgetScatterPlot = React.memo((
// Perform unit conversion, report errors
const data = useMemo(() => {
if (!dataRaw) return
- const x = xAxis.type === DType.Timestamp
+ const x = xAxis.dtype === DType.Timestamp
? dataRaw.x
: new Quantity(dataRaw.x, storageUnitX).to(xAxis.unit).value()
- const y = yAxis.type === DType.Timestamp
+ const y = yAxis.dtype === DType.Timestamp
? dataRaw.y
: new Quantity(dataRaw.y, storageUnitY).to(yAxis.unit).value()
const color = dataRaw.color && (discrete
@@ -275,7 +275,7 @@ export const WidgetScatterPlot = React.memo((
: new Quantity(dataRaw.color, storageUnitColor).to(colorAxis.unit).value()
)
return {x, y, color, id: dataRaw.id}
- }, [dataRaw, xAxis.type, xAxis.unit, storageUnitX, yAxis.type, yAxis.unit, storageUnitY, discrete, storageUnitColor, colorAxis.unit])
+ }, [dataRaw, xAxis.dtype, xAxis.unit, storageUnitX, yAxis.dtype, yAxis.unit, storageUnitY, discrete, storageUnitColor, colorAxis.unit])
const handleEdit = useCallback(() => {
setWidget(old => { return {...old, editing: true } })
diff --git a/nomad/search.py b/nomad/search.py
index 6eef2d16e8988604c81a1ee5950e75c8850cfa33..1aeac19c2f6557f725ddf43e67a32bfddcfe8dd9 100644
--- a/nomad/search.py
+++ b/nomad/search.py
@@ -32,11 +32,8 @@ update the v1 materials index according to the performed changes. TODO this is o
partially implemented.
"""
-import fnmatch
import json
import math
-import re
-import sys
from enum import Enum
from typing import (
Any,
@@ -47,7 +44,6 @@ from typing import (
Iterator,
List,
Optional,
- Sequence,
Tuple,
Union,
cast,
@@ -483,35 +479,6 @@ def _es_to_entry_dict(
Translates an ES hit response into a response data object that is expected
by the API.
"""
-
- def filter_hit(hit, include, exclude):
- """Used to filter the hit based on the required fields."""
- flattened_dict = utils.flatten_dict(hit, flatten_list=True)
- keys = list(flattened_dict.keys())
- key_pattern_map = {key: re.sub(r'\.\d+\.', '.', key) for key in keys}
- if include is not None:
- keys = [
- key
- for key in keys
- if any(
- fnmatch.fnmatch(key_pattern_map[key], pattern)
- for pattern in include
- )
- ]
- if exclude is not None:
- keys = [
- key
- for key in keys
- if not any(
- fnmatch.fnmatch(key_pattern_map[key], pattern)
- for pattern in exclude
- )
- ]
- filtered_dict = {key: flattened_dict[key] for key in keys}
- hit = utils.rebuild_dict(filtered_dict)
-
- return hit
-
entry_dict = hit.to_dict()
# Add metadata default values
@@ -602,7 +569,7 @@ def _es_to_entry_dict(
if required.exclude
else None
)
- entry_dict = filter_hit(entry_dict, include_patterns, exclude_patterns)
+ entry_dict = utils.prune_dict(entry_dict, include_patterns, exclude_patterns)
return entry_dict
diff --git a/nomad/utils/__init__.py b/nomad/utils/__init__.py
index 1be80bfa98ba69ac8351e8f131e88e03e18647fa..757b6670737ba3e97f39879ad0706a9005464961 100644
--- a/nomad/utils/__init__.py
+++ b/nomad/utils/__init__.py
@@ -42,6 +42,7 @@ from typing import List, Iterable, Union, Any, Dict
from collections import OrderedDict
from functools import reduce
from itertools import takewhile
+import fnmatch
import base64
from contextlib import contextmanager
import json
@@ -700,6 +701,108 @@ def rebuild_dict(src: dict, separator: str = '.'):
return ret
+def prune_dict(data, include_patterns=None, exclude_patterns=None):
+ """
+ Prune a nested dictionary based on include and exclude branch patterns.
+
+ Args:
+ data: The nested dictionary to prune.
+ include_patterns: List of branch patterns to include. Supports wildcards
+ like `root.child.*`.
+ exclude_patterns: List of branch patterns to exclude. Supports wildcards
+ like `root.child.*`.
+ parent_key: Used internally for recursion to track the full key path.
+
+ Returns:
+ Pruned dictionary.
+ """
+
+ # Preprocess patterns
+ def process_patterns(patterns):
+ if patterns is None:
+ return []
+ processed = []
+ for pattern in patterns:
+ if pattern.endswith('*'):
+ parts = pattern.rstrip('*').removesuffix('.').split('.')
+ processed.append((parts, True))
+ else:
+ parts = pattern.split('.')
+ processed.append((parts, False))
+ return processed
+
+ include_patterns = process_patterns(include_patterns)
+ exclude_patterns = process_patterns(exclude_patterns)
+
+ def matches(path, patterns):
+ for pattern_parts, wildcard in patterns:
+ if wildcard:
+ if path[: len(pattern_parts)] == tuple(pattern_parts):
+ return True
+ else:
+ if path == tuple(pattern_parts):
+ return True
+ return False
+
+ def has_descendant_pattern(path, patterns):
+ for pattern_parts, _ in patterns:
+ if (
+ len(pattern_parts) > len(path)
+ and tuple(pattern_parts[: len(path)]) == path
+ ):
+ return True
+ return False
+
+ def prune(data, path=()):
+ # Check exclude patterns
+ if matches(path, exclude_patterns):
+ return None # Excluded
+
+ # Check include patterns
+ if include_patterns:
+ if matches(path, include_patterns):
+ include_current = True
+ elif has_descendant_pattern(path, include_patterns):
+ include_current = False # Need to check deeper
+ else:
+ return None # Excluded
+ else:
+ include_current = True
+
+ if isinstance(data, dict):
+ new_dict = {}
+ for key, value in data.items():
+ new_path = path + (str(key),)
+ pruned_value = prune(value, new_path)
+ if pruned_value is not None:
+ new_dict[key] = pruned_value
+ if new_dict:
+ return new_dict
+ elif include_patterns and matches(path, include_patterns):
+ return {} # Include empty dict if explicitly included
+ else:
+ return None # Exclude empty dicts
+ elif isinstance(data, list):
+ new_list = []
+ for item in data:
+ pruned_item = prune(item, path)
+ if pruned_item is not None:
+ new_list.append(pruned_item)
+ if new_list:
+ return new_list
+ elif include_patterns and matches(path, include_patterns):
+ return [] # Include empty list if explicitly included
+ else:
+ return None # Exclude empty lists
+ else:
+ if include_current:
+ return data
+ else:
+ return None
+
+ return prune(data) or {}
+
+
def deep_get(dictionary, *keys):
"""
Helper that can be used to access nested dictionary-like containers using a
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 91ca1f2c6aca9cbef408007c0045507c052f8d23..1853751a0b4e5c4d6cea5de11ae5e5ed1fe5580f 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -30,6 +30,7 @@ from nomad.utils import (
structlogging,
flatten_dict,
rebuild_dict,
+ prune_dict,
deep_get,
dict_to_dataframe,
dataframe_to_dict,
@@ -167,6 +168,79 @@ def test_dict_flatten_rebuild(data, flatten_list):
assert rebuilt == data
+@pytest.mark.parametrize(
+ 'data, include, exclude, expected',
+ [
+ pytest.param(
+ {'a': 1, 'b': {'c': 2}, 'd': [{'e': 3}, {'f': 4}]},
+ None,
+ None,
+ {'a': 1, 'b': {'c': 2}, 'd': [{'e': 3}, {'f': 4}]},
+ id='all',
+ ),
+ pytest.param(
+ {'a': 1, 'b': {'c': 2}, 'd': [{'e': 3}, {'f': 4}]},
+ ['d.f'],
+ None,
+ {'d': [{'f': 4}]},
+ id='include exact',
+ ),
+ pytest.param(
+ {'a': 1, 'b': {'c': 2}, 'd': [{'e': 3}, {'f': 4}]},
+ ['d.*'],
+ None,
+ {'d': [{'e': 3}, {'f': 4}]},
+ id='include wildcard',
+ ),
+ pytest.param(
+ {'a': 1, 'b': {'c': 2}, 'd': [{'e': 3}, {'f': 4}]},
+ ['d*'],
+ None,
+ {'d': [{'e': 3}, {'f': 4}]},
+ id='include wildcard no dot',
+ ),
+ pytest.param(
+ {'a': 1, 'b': {'c': 2}, 'd': [{'e': 3}, {'f': 4}]},
+ None,
+ ['d.f'],
+ {'a': 1, 'b': {'c': 2}, 'd': [{'e': 3}]},
+ id='exclude exact',
+ ),
+ pytest.param(
+ {'a': 1, 'b': {'c': 2}, 'd': [{'e': 3}, {'f': 4}]},
+ None,
+ ['d.*'],
+ {'a': 1, 'b': {'c': 2}},
+ id='exclude wildcard',
+ ),
+ pytest.param(
+ {'a': {'b': 1}},
+ None,
+ ['a*'],
+ {},
+ id='exclude wildcard no dot',
+ ),
+ pytest.param(
+ {'a': {'b': 1}},
+ None,
+ ['a.*'],
+ {},
+ id='exclude everything',
+ ),
+ pytest.param(
+ {'a': [], 'b': {}},
+ ['a', 'b'],
+ None,
+ {'a': [], 'b': {}},
+ id='preserve empty structures if they are included',
+ ),
+ ],
+)
+def test_prune_dict(data, include, exclude, expected):
+ pruned = prune_dict(data, include, exclude)
+ assert pruned == expected
+
+
@pytest.mark.parametrize(
'data, path, value, exception',
[