From bef33475c794afe9328b0e38e9986ae05be78aaf Mon Sep 17 00:00:00 2001
From: Sandor Brockhauser <sandor.brockhauser@physik.hu-berlin.de>
Date: Mon, 10 Feb 2025 10:37:38 +0000
Subject: [PATCH] Added support for targeting any NeXus search quantity in
aggregations by providing the data type at the end of the search quantity
name.
Changelog: Added
---
gui/src/components/plotting/common.js | 81 +++++++------
gui/src/components/search/Filter.js | 9 +-
gui/src/components/search/FilterRegistry.js | 41 ++++++-
gui/src/components/search/SearchContext.js | 108 +++++++++++-------
.../components/search/input/InputHistogram.js | 18 ++-
.../components/search/input/InputMetainfo.js | 23 +++-
.../search/widgets/WidgetHistogram.js | 1 +
.../search/widgets/WidgetScatterPlot.js | 4 +-
.../search/widgets/WidgetScatterPlotEdit.js | 2 +-
.../components/search/widgets/WidgetTerms.js | 10 +-
gui/src/utils.js | 20 +++-
nomad/metainfo/elasticsearch_extension.py | 20 ++--
nomad/metainfo/metainfo.py | 51 ++++-----
nomad/search.py | 26 +----
tests/fixtures/data.py | 38 ------
15 files changed, 249 insertions(+), 203 deletions(-)
diff --git a/gui/src/components/plotting/common.js b/gui/src/components/plotting/common.js
index 1dd7a4a179..dd67e862f1 100644
--- a/gui/src/components/plotting/common.js
+++ b/gui/src/components/plotting/common.js
@@ -41,8 +41,9 @@ import {
eachQuarterOfInterval
} from 'date-fns'
import { scale as chromaScale } from 'chroma-js'
-import { scale as scaleUtils, add, DType, formatNumber, getDisplayLabel, parseJMESPath } from '../../utils.js'
+import { scale as scaleUtils, add, DType, formatNumber, getDisplayLabel, parseQuantityName, parseJMESPath, getLabel } from '../../utils.js'
import { Unit } from '../units/Unit'
+import { postFixMap } from '../search/FilterRegistry.js'
export const scales = {
'linear': 'linear',
@@ -497,43 +498,49 @@ export function getPlotTracesVertical(plots, theme) {
* @param {object} units Units in current unit system.
*/
export function getAxisConfig(axis, filterData, units) {
- const {quantity} = parseJMESPath(axis?.search_quantity)
- const filter = filterData[quantity]
- const dtype = filter?.dtype
- const unit = axis.unit
- ? new Unit(axis.unit)
- : new Unit(filter?.unit || 'dimensionless').toSystem(units)
-
- // Create the final label
- let title = axis.title || filter?.label || getDisplayLabel(filter)
- let finalUnit
- if (unit) {
- finalUnit = new Unit(unit).label()
- } else if (filter?.unit) {
- finalUnit = new Unit(filter.unit).toSystem(units).label()
- }
- if (finalUnit) {
- title = `${title} (${finalUnit})`
+ const {quantity} = parseJMESPath(axis?.search_quantity)
+ const filter = filterData[quantity]
+ let dtype = filter?.dtype
+ if (!dtype || dtype === DType.Unknown) {
+ const dtypeFromName = postFixMap[parseQuantityName(quantity)?.dtype]
+ if (dtypeFromName) {
+ dtype = dtypeFromName
}
+ }
+ const unit = axis.unit
+ ? new Unit(axis.unit)
+ : new Unit(filter?.unit || 'dimensionless').toSystem(units)
+
+ // Create the final label
+ let title = axis.title || filter?.label || getDisplayLabel(filter) || getLabel(quantity)
+ let finalUnit
+ if (unit) {
+ finalUnit = new Unit(unit).label()
+ } else if (filter?.unit) {
+ finalUnit = new Unit(filter.unit).toSystem(units).label()
+ }
+ if (finalUnit) {
+ title = `${title} (${finalUnit})`
+ }
- // Determine the final description
- let description = axis.description || filter?.description || ''
- if (description && quantity) {
- description = (
- <>
- <Typography>{title}</Typography>
- <b>Description: </b>{description}<br/>
- <b>Path: </b>{quantity}
- </>
- )
- }
+ // Determine the final description
+ let description = axis.description || filter?.description || ''
+ if (description && quantity) {
+ description = (
+ <>
+ <Typography>{title}</Typography>
+ <b>Description: </b>{description}<br/>
+ <b>Path: </b>{quantity}
+ </>
+ )
+ }
- return {
- ...axis,
- description,
- title,
- unit,
- dtype,
- quantity
- }
+ return {
+ ...axis,
+ description,
+ title,
+ unit,
+ dtype,
+ quantity
+ }
}
diff --git a/gui/src/components/search/Filter.js b/gui/src/components/search/Filter.js
index fb518d9a4a..dd4d4782e0 100644
--- a/gui/src/components/search/Filter.js
+++ b/gui/src/components/search/Filter.js
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { isNil, isArray, isEmpty, capitalize, split } from 'lodash'
+import { isNil, isArray, isEmpty } from 'lodash'
import { searchQuantities } from '../../config'
import {
getDatatype,
@@ -24,7 +24,7 @@ import {
getDisplayLabel,
DType,
multiTypes,
- parseQuantityName
+ getLabel
} from '../../utils'
import { Unit } from '../units/Unit'
@@ -140,11 +140,6 @@ export class Filter {
this.description = params?.description || def?.description
this.unit = params?.unit || def?.unit
this.dimension = def?.unit ? new Unit(def?.unit).dimension() : 'dimensionless'
- function getLabel(quantity) {
- if (isNil(quantity)) return ''
- const {path} = parseQuantityName(quantity)
- return capitalize(split(path, '.').slice(-1)[0].replace(/_/g, ' '))
- }
this.label = params?.label || getDisplayLabel(def) || getLabel(this.quantity)
this.parent = parent
this.group = params.group
diff --git a/gui/src/components/search/FilterRegistry.js b/gui/src/components/search/FilterRegistry.js
index 0f5755f9e3..4d91c63e73 100644
--- a/gui/src/components/search/FilterRegistry.js
+++ b/gui/src/components/search/FilterRegistry.js
@@ -36,6 +36,13 @@ const dtypeMap = {
[DType.Enum]: 'str',
[DType.Boolean]: 'bool'
}
+export const postFixMap = {
+ 'int': DType.Int,
+ 'float': DType.Float,
+ 'datetime': DType.Timestamp,
+ 'str': DType.String,
+ 'bool': DType.Boolean
+}
/**
* This function is used to register a new filter within the SearchContext.
@@ -732,6 +739,28 @@ export function getStaticSuggestions(quantities, filterData) {
return suggestions
}
+// add a specific filter
+export function addFilter(filterPath, def, repeats, filtersData, setFiltersData) {
+ if (filterPath in filtersData) {
+ return
+ }
+ const newFilters = {}
+ const pathDtype = filterPath.split(schemaSeparator).slice(-1)[0]
+ const dtype = dtypeMap[getDatatype(def)] || pathDtype
+ // TODO: For some Nexus quantities, the data types cannot be fetched.
+ if (!dtype) {
+ return
+ }
+ const params = {
+ name: filterPath,
+ quantity: filterPath,
+ aggregatable: new Set([DType.String, DType.Enum, DType.Boolean]).has(getDatatype(def)),
+ repeats: repeats
+ }
+ newFilters[filterPath] = new Filter(def, params)
+ setFiltersData((old) => ({...old, ...newFilters}))
+}
+
/**
* HOC that is used to preload search quantities from all required schemas. This
* simplifies the rendering logic by first loading all schemas before rendering
@@ -744,7 +773,7 @@ export const withSearchQuantities = (WrappedComponent) => {
const [yamlOptions, nexusOptions, initialFilterData] = useMemo(() => {
const options = getOptions(initialSearchQuantities)
const yamlOptions = options.filter((name) => name.includes(`#${yamlSchemaPrefix}`))
- const nexusOptions = options.filter((name) => name.startsWith('nexus.'))
+ const nexusOptions = options.filter((name) => name.includes(`#pynxtools.nomad.schema`))
// Perform glob filtering on default filters. Only exclude affects the
// default filters.
@@ -786,14 +815,14 @@ export const withSearchQuantities = (WrappedComponent) => {
// Nexus metainfo is loaded here once metainfo is ready
useEffect(() => {
if (!nexusOptions.length || !metainfo) return
- const pkg = metainfo._packageDefs['nexus']
+ const pkg = metainfo._packageDefs['pynxtools.nomad.schema']
const sections = pkg.section_definitions
const nexusFilters = {}
for (const section of sections) {
- const sectionPath = `nexus.${section.name}`
+ const sectionPath = `data.${section.name}`
// The NeXus section is skipped (it contains duplicate information)
- if (sectionPath === 'nexus.NeXus') continue
+ if (sectionPath === 'data.NeXus') continue
// Only applications definitions are loaded
if (section?.more?.nx_category !== 'application') continue
@@ -806,8 +835,8 @@ export const withSearchQuantities = (WrappedComponent) => {
// Add all included quantities recursively
for (const [def, path, repeats] of getQuantities(section)) {
const filterPath = `${sectionPath}.${path}`
- const included = glob(filterPath, initialSearchQuantities?.include, initialSearchQuantities?.exclude)
- if (!included) continue
+ // const included = glob(filterPath, initialSearchQuantities?.include, initialSearchQuantities?.exclude)
+ // if (!included) continue
const dtype = dtypeMap[getDatatype(def)]
// TODO: For some Nexus quantities, the data types cannot be fetched.
if (!dtype) {
diff --git a/gui/src/components/search/SearchContext.js b/gui/src/components/search/SearchContext.js
index c54c7457ac..dcffd4fce2 100644
--- a/gui/src/components/search/SearchContext.js
+++ b/gui/src/components/search/SearchContext.js
@@ -72,7 +72,7 @@ import UploadStatusIcon from '../uploads/UploadStatusIcon'
import { getWidgetsObject } from './widgets/Widget'
import { inputSectionContext } from './input/InputNestedObject'
import { SearchSuggestion } from './SearchSuggestion'
-import { withSearchQuantities } from './FilterRegistry'
+import { withSearchQuantities, addFilter } from './FilterRegistry'
import { useUnitContext } from '../units/UnitContext'
const useWidthConstrainedStyles = makeStyles(theme => ({
@@ -442,6 +442,7 @@ export const SearchContextRaw = React.memo(({
useSetPagination,
usePaginationState,
useAgg,
+ useRemoveAgg,
useAggs,
useSetAggsResponse,
useSetFilters,
@@ -1088,13 +1089,16 @@ export const SearchContextRaw = React.memo(({
const key = useMemo(() => `${name}:${id}`, [name, id])
const setAgg = useSetRecoilState(aggsFamily(key))
const aggResponse = useRecoilValue(aggsResponseFamily(key))
- const filtersData = useRecoilValue(filtersDataState)
+ const [filtersData, setFiltersData] = useRecoilState(filtersDataState)
// Whenever the aggregation requirements change, create the final
// aggregation config and set it in the search context: this will then
// trigger any required API calls that also return the aggregation
// response that is returned by this hook.
useEffect(() => {
+ if (!(name in filtersData)) {
+ addFilter(name, "", true, filtersData, setFiltersData)
+ }
const defaults = filtersData[name]?.aggs?.[config?.type]
const finalConfig = {
update: update,
@@ -1102,11 +1106,22 @@ export const SearchContextRaw = React.memo(({
...config
}
setAgg(finalConfig)
- }, [name, update, setAgg, config, filtersData])
+ }, [name, update, setAgg, config, filtersData, setFiltersData])
return aggResponse
}
+ /**
+ * Function for removing an agg from use. Should be called when a component
+ * that uses an agg is unmounted.
+ */
+ const useRemoveAgg = () => {
+ return useRecoilCallback(({ reset }) => (name, id) => {
+ const key = `${name}:${id}`
+ reset(aggsFamilyRaw(key))
+ }, [])
+ }
+
/**
* This hook will expose a function for setting the values of all filters.
*
@@ -1165,6 +1180,7 @@ export const SearchContextRaw = React.memo(({
useRemoveWidget,
useResetWidgets,
useAgg,
+ useRemoveAgg,
useSetFilters,
useUpdateFilter,
useQuery,
@@ -1234,31 +1250,37 @@ export const SearchContextRaw = React.memo(({
*/
const resolve = useCallback(prop => {
const {response, timestamp, queryChanged, paginationChanged, search, aggsToUpdate, resource, callbackAgg, callbackHits} = prop
- const data = response.response
+ const error = !response
const next = apiQueue.current[0]
if (next !== timestamp) {
apiMap.current[timestamp] = prop
return
}
- // Update the aggregations if new aggregation data is received. The old
- // aggregation data is preserved and new information is updated.
- if (!isEmpty(data.aggregations)) {
- const newAggs = convertAggAPIToGUI(data.aggregations, aggsToUpdate, resource, filtersData)
- callbackAgg && callbackAgg(newAggs, undefined, true)
+ if (error) {
+ callbackAgg && callbackAgg(undefined, error, true)
+ callbackHits && callbackHits(undefined, error, true, undefined)
} else {
- callbackAgg && callbackAgg(undefined, undefined, false)
- }
- // Update the query results if new data is received.
- if (queryChanged || paginationChanged) {
- paginationResponse.current = data.pagination
- const newResults = {
- response: response,
- pagination: combinePagination(search.pagination, data.pagination),
- setPagination: setPagination
+ const data = response.response
+ // Update the aggregations if new aggregation data is received. The old
+ // aggregation data is preserved and new information is updated.
+ if (!isEmpty(data.aggregations)) {
+ const newAggs = convertAggAPIToGUI(data.aggregations, aggsToUpdate, resource, filtersData)
+ callbackAgg && callbackAgg(newAggs, undefined, true)
+ } else {
+ callbackAgg && callbackAgg(undefined, undefined, false)
+ }
+ // Update the query results if new data is received.
+ if (queryChanged || paginationChanged) {
+ paginationResponse.current = data.pagination
+ const newResults = {
+ response: response,
+ pagination: combinePagination(search.pagination, data.pagination),
+ setPagination: setPagination
+ }
+ callbackHits && callbackHits(newResults, undefined, true, search)
+ } else {
+ callbackHits && callbackHits(undefined, undefined, false, search)
}
- callbackHits && callbackHits(newResults, undefined, true, search)
- } else {
- callbackHits && callbackHits(undefined, undefined, false, search)
}
// Remove this query from queue and see if next can be resolved.
apiQueue.current.shift()
@@ -1373,24 +1395,30 @@ export const SearchContextRaw = React.memo(({
const timestamp = Date.now()
apiQueue.current.push(timestamp)
setApiQuery(search?.query)
+ const resolveArgs = {
+ timestamp,
+ queryChanged,
+ paginationChanged,
+ search,
+ aggsToUpdate,
+ resource,
+ callbackAgg,
+ callbackHits
+ }
api.query(resource, search, {loadingIndicator: true, returnRequest: true})
- .then((response) => {
- return resolve({
- response,
- timestamp,
- queryChanged,
- paginationChanged,
- search,
- aggsToUpdate,
- resource,
- callbackAgg,
- callbackHits
- })
- })
+ .then((response) => resolve({response, ...resolveArgs}))
.catch((error) => {
- raiseError(error)
- callbackAgg && callbackAgg(undefined, error, true)
- callbackHits && callbackHits(undefined, error, true, undefined)
+ let apiErrorMessage = error?.apiMessage?.[0]?.msg
+ if (apiErrorMessage) {
+ if (apiErrorMessage?.endsWith('is not a doc quantity')) {
+ const name = apiErrorMessage.split(' ')[0]
+ apiErrorMessage = `Could not find definition for the search quantity "${name}". Please remove it from the search and try again.`
+ }
+ error = Error(apiErrorMessage)
+ error.name = 'BadRequest'
+ raiseError(error)
+ }
+ resolve({undefined, ...resolveArgs})
})
}, [filtersData, filterDefaults, filtersLocked, resource, api, raiseError, resolve, dynamicQueryModes, setApiQuery])
@@ -1715,6 +1743,7 @@ export const SearchContextRaw = React.memo(({
useResults,
useHits,
useAgg,
+ useRemoveAgg,
useAggCall,
useSetFilters,
useUpdateFilter
@@ -1760,6 +1789,7 @@ export const SearchContextRaw = React.memo(({
useApiQuery,
useApiData,
useAgg,
+ useRemoveAgg,
useAggs,
useQuery,
useSetFilters,
@@ -2112,7 +2142,7 @@ function convertAggGUIToAPI(aggs, resource, filtersData) {
for (const [key, agg] of Object.entries(aggs)) {
const filterName = rsplit(key, ':', 1)[0]
if (agg.update) {
- const exclusive = filtersData[filterName].exclusive
+ const exclusive = filterName in filtersData && filtersData[filterName].exclusive
const type = agg.type
const apiAgg = apiAggs[key] || {}
const aggSet = agg.set
@@ -2266,7 +2296,7 @@ function reduceAggs(aggs, oldAggs, queryChanged, updatedFilters, filtersData) {
// If the filter is exclusive, and ONLY it has been modified in this
// query, we do not update it's aggregation.
const exclude = isNil(agg.exclude_from_search)
- ? filtersData[filter_name].exclusive
+ ? filter_name in filtersData && filtersData[filter_name].exclusive
: agg.exclude_from_search
if (exclude && updatedFilters.has(filter_name) && updatedFilters.size === 1) {
update = false
diff --git a/gui/src/components/search/input/InputHistogram.js b/gui/src/components/search/input/InputHistogram.js
index d90e7e5b71..1a9fd25933 100644
--- a/gui/src/components/search/input/InputHistogram.js
+++ b/gui/src/components/search/input/InputHistogram.js
@@ -59,7 +59,7 @@ export const Histogram = React.memo(({
classes,
'data-testid': testID
}) => {
- const {filterData, useAgg, useFilterState} = useSearchContext()
+ const {filterData, useAgg, useRemoveAgg, useFilterState} = useSearchContext()
const sectionContext = useContext(inputSectionContext)
const repeats = sectionContext?.repeats
const styles = useStyles({classes})
@@ -80,6 +80,18 @@ export const Histogram = React.memo(({
const [maxInclusive, setMaxInclusive] = useState(true)
const highlight = Boolean(filter)
+ // When component is unmounted, remove aggregation requests
+ const quantityName = x.search_quantity
+ const aggIdHistogram = `${aggId}_histogram`
+ const aggIdSlider = `${aggId}_slider`
+ const removeAgg = useRemoveAgg()
+ useEffect(() => {
+ return () => {
+ removeAgg(quantityName, aggIdHistogram)
+ removeAgg(quantityName, aggIdSlider)
+ }
+ }, [removeAgg, quantityName, aggIdHistogram, aggIdSlider])
+
// Determine the description and units
const def = filterData[x.search_quantity]
const unitStorage = useMemo(() => { return new Unit(def?.unit || 'dimensionless') }, [def])
@@ -161,7 +173,7 @@ export const Histogram = React.memo(({
? {type: 'histogram', buckets: nBins, exclude_from_search, extended_bounds}
: {type: 'histogram', interval: discretization, exclude_from_search, extended_bounds}
}, [filter, fromDisplayUnit, isTime, discretization, nBins, autorange])
- const agg = useAgg(x.search_quantity, visible && showStatistics, `${aggId}_histogram`, aggHistogramConfig)
+ const agg = useAgg(quantityName, visible && showStatistics, aggIdHistogram, aggHistogramConfig)
useEffect(() => {
if (!isNil(agg)) {
firstLoad.current = false
@@ -171,7 +183,7 @@ export const Histogram = React.memo(({
// Aggregation when the statistics are disabled: a simple min_max aggregation
// is enough in order to get the slider range.
const aggSliderConfig = useMemo(() => ({type: 'min_max', exclude_from_search: true}), [])
- const aggSlider = useAgg(x.search_quantity, visible && !showStatistics, `${aggId}_slider`, aggSliderConfig)
+ const aggSlider = useAgg(quantityName, visible && !showStatistics, aggIdSlider, aggSliderConfig)
// Determine the global minimum and maximum values
const [minGlobal, maxGlobal] = useMemo(() => {
diff --git a/gui/src/components/search/input/InputMetainfo.js b/gui/src/components/search/input/InputMetainfo.js
index 4263809e38..3eced926ca 100644
--- a/gui/src/components/search/input/InputMetainfo.js
+++ b/gui/src/components/search/input/InputMetainfo.js
@@ -27,11 +27,14 @@ import React, {
import { makeStyles } from '@material-ui/core/styles'
import PropTypes from 'prop-types'
import { Tooltip, List, ListItemText, ListSubheader } from '@material-ui/core'
+import { has } from 'lodash'
import HelpOutlineIcon from '@material-ui/icons/HelpOutline'
import { getSchemaAbbreviation, getSuggestions, parseJMESPath } from '../../../utils'
import { useSearchContext } from '../SearchContext'
import { VariableSizeList } from 'react-window'
import { InputText } from './InputText'
+import { dtypeSeparator } from '../../../config'
+import { postFixMap } from '../FilterRegistry'
/**
* Wrapper around InputText that is specialized in showing metainfo options. The
@@ -86,13 +89,15 @@ export const InputMetainfoControlled = React.memo(({
}
if (validate) {
return validate(value)
- } else if (!(keysSet.has(value))) {
+ // Check if the quantity is available. The check is ignored if the quantity
+ // explicitly contains the data type as a postfix.
+ } else if (!(keysSet.has(value)) && !hasDatatype(value)) {
return {valid: false, error: `The quantity "${value}" is not available.`}
}
return {valid: true, error: undefined}
}, [validate, keysSet, optional, disableValidation])
- // Handles the selectance of a suggested value
+ // Handles the selection of a suggested value
const handleSelect = useCallback((key) => {
onSelect?.(key, options[key])
}, [onSelect, options])
@@ -223,15 +228,15 @@ export const InputJMESPath = React.memo(React.forwardRef(({
if (errorParse) {
return {valid: false, error: 'Invalid JMESPath query, please check your syntax.'}
}
- if (!(keysSet.has(quantity))) {
+ if (!(keysSet.has(quantity)) && !hasDatatype(quantity)) {
return {valid: false, error: `The quantity "${quantity}" is not available.`}
}
for (const extra of extras) {
- if (!filterData[extra]) {
+ if (!filterData[extra] && !hasDatatype(extra)) {
return {valid: false, error: `The quantity "${extra}" is not available.`}
}
}
- if (filterData[quantity].repeats_section && quantity === path + schema) {
+ if (filterData[quantity]?.repeats_section && quantity === path + schema) {
return {valid: false, error: `The quantity "${quantity}" is contained in at least one repeatable section. Please use JMESPath syntax to select one or more target sections.`}
}
return {valid: true, error: undefined}
@@ -436,3 +441,11 @@ function getMetainfoOptions(filterData, dtypes, dtypesRepeatable, disableNonAggr
}])
)
}
+
+/**
+ * Checks if the given quantity name contains the data type as a postfix.
+*/
+function hasDatatype(quantity) {
+ const postFix = quantity.split(dtypeSeparator).pop()
+ return has(postFixMap, postFix)
+}
diff --git a/gui/src/components/search/widgets/WidgetHistogram.js b/gui/src/components/search/widgets/WidgetHistogram.js
index bf61008469..f14a8a36bb 100644
--- a/gui/src/components/search/widgets/WidgetHistogram.js
+++ b/gui/src/components/search/widgets/WidgetHistogram.js
@@ -45,6 +45,7 @@ export const WidgetHistogram = React.memo((
const setWidget = useSetWidget(id)
// Create final axis config for the plot
+ // console.log(x)
const xAxis = useMemo(() => getAxisConfig(x, filterData, units), [x, filterData, units])
const handleEdit = useCallback(() => {
diff --git a/gui/src/components/search/widgets/WidgetScatterPlot.js b/gui/src/components/search/widgets/WidgetScatterPlot.js
index 2e403e6f85..c8bf11c6b7 100644
--- a/gui/src/components/search/widgets/WidgetScatterPlot.js
+++ b/gui/src/components/search/widgets/WidgetScatterPlot.js
@@ -107,8 +107,8 @@ export const WidgetScatterPlot = React.memo((
// Get storage unit for API communication
const {storageUnitX, storageUnitY, storageUnitColor} = useMemo(() => {
if (error) return {}
- const storageUnitX = new Unit(filterData[xParsed.quantity].unit || 'dimensionless')
- const storageUnitY = new Unit(filterData[yParsed.quantity].unit || 'dimensionless')
+ const storageUnitX = new Unit(filterData[xParsed.quantity]?.unit || 'dimensionless')
+ const storageUnitY = new Unit(filterData[yParsed.quantity]?.unit || 'dimensionless')
const storageUnitColor = new Unit(filterData[colorParsed?.quantity]?.unit || 'dimensionless')
return {storageUnitX, storageUnitY, storageUnitColor}
}, [filterData, xParsed.quantity, yParsed.quantity, colorParsed?.quantity, error])
diff --git a/gui/src/components/search/widgets/WidgetScatterPlotEdit.js b/gui/src/components/search/widgets/WidgetScatterPlotEdit.js
index 05f8fe9a1f..e0c7dad5f0 100644
--- a/gui/src/components/search/widgets/WidgetScatterPlotEdit.js
+++ b/gui/src/components/search/widgets/WidgetScatterPlotEdit.js
@@ -93,7 +93,7 @@ export const WidgetScatterPlotEdit = React.memo(({widget}) => {
const handleAcceptQuantity = useCallback((key, value) => {
handleAccept(key, value)
const { quantity } = parseJMESPath(value)
- const dimension = filterData[quantity]?.dimension
+ const dimension = filterData[quantity]?.dimension || 'dimensionless'
setDimensions((old) => ({...old, [key]: dimension}))
}, [handleAccept, filterData])
diff --git a/gui/src/components/search/widgets/WidgetTerms.js b/gui/src/components/search/widgets/WidgetTerms.js
index e110b34791..b2b550e199 100644
--- a/gui/src/components/search/widgets/WidgetTerms.js
+++ b/gui/src/components/search/widgets/WidgetTerms.js
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import React, { useState, useCallback, useMemo } from 'react'
+import React, { useState, useCallback, useMemo, useEffect } from 'react'
import PropTypes from 'prop-types'
import { string, bool } from 'yup'
import clsx from 'clsx'
@@ -108,12 +108,18 @@ export const WidgetTerms = React.memo((
className,
'data-testid': testID
}) => {
- const {useAgg, useFilterState, filterData} = useSearchContext()
+ const {useAgg, useRemoveAgg, useFilterState, filterData} = useSearchContext()
const styles = useStyles()
const [filter, setFilter] = useFilterState(search_quantity)
const { height, ref } = useResizeDetector()
const { useSetWidget } = useSearchContext()
const setWidget = useSetWidget(id)
+ const removeAgg = useRemoveAgg()
+
+ // When component is unmounted, remove aggregation request
+ useEffect(() => {
+ return () => removeAgg(search_quantity, id)
+ }, [removeAgg, search_quantity, id])
// The terms aggregations need to request at least 1 item or an API error is thrown
const aggSize = useMemo(() => Math.max(Math.floor(height / inputItemHeight), 1), [height])
diff --git a/gui/src/utils.js b/gui/src/utils.js
index d3fc8e7644..c9c198a68a 100644
--- a/gui/src/utils.js
+++ b/gui/src/utils.js
@@ -614,7 +614,16 @@ export function getSerializer(dtype, pretty = true) {
} else if (dtype === DType.Boolean) {
return (value) => value ? 'true' : 'false'
} else {
- return (value) => value
+ return (value) => {
+ if (isNil(value)) {
+ return value
+ }
+ let val = value instanceof Quantity ? value.normalized_value : value
+ if (isNumber(val) && pretty) {
+ val = formatNumber(val)
+ }
+ return val
+ }
}
}
@@ -1546,6 +1555,15 @@ export function parseQuantityName(fullName) {
return {path, schema: schemaNew, dtype}
}
+/**
+ * Autogenerates a sensible default name for a quantity based on its path.
+ */
+export function getLabel(quantity) {
+ if (isNil(quantity)) return ''
+ const {path} = parseQuantityName(quantity)
+ return capitalize(split(path, '.').slice(-1)[0].replace(/_/g, ' '))
+}
+
/**
* Used to parse a possible query operator from a quantity name.
*/
diff --git a/nomad/metainfo/elasticsearch_extension.py b/nomad/metainfo/elasticsearch_extension.py
index f3a3538200..6c44725683 100644
--- a/nomad/metainfo/elasticsearch_extension.py
+++ b/nomad/metainfo/elasticsearch_extension.py
@@ -198,7 +198,7 @@ if TYPE_CHECKING:
schema_separator = '#'
dtype_separator = '#'
yaml_prefix = 'entry_id:'
-nexus_prefix = 'pynxtools.nomad.schema.NeXus.'
+nexus_prefix = 'pynxtools.nomad.schema'
class DocumentType:
@@ -488,8 +488,6 @@ class DocumentType:
if self != entry_type:
return None
- logger = utils.get_logger(__name__, doc_type=self.name)
-
# Remove existing dynamic quantities
for name, quantity in list(self.quantities.items()):
if quantity.dynamic:
@@ -556,7 +554,7 @@ class DocumentType:
section.section_cls, EntryData
):
schema_name = section.qualified_name()
- if section.name == 'NeXus':
+ if schema_name.startswith('pynxtools'):
# Allow App searches for specific AppDefs:
selected_path = [
'Root',
@@ -569,7 +567,11 @@ class DocumentType:
'Ellipsometry',
'Raman',
]
- max_level = 3
+ if section.name in selected_path:
+ max_level = 3
+ else:
+ max_level = 0
+ selected_path = ['']
else:
selected_path = ['']
max_level = -1
@@ -596,14 +598,6 @@ class DocumentType:
annotation, qualified_name=full_name, repeats=repeats
)
quantities_dynamic[full_name] = search_quantity
- # print(
- # 'With '
- # + section.name
- # + '.'
- # + path_prefix
- # + ' the number of dynamic quantities: '
- # + str(len(quantities_dynamic))
- # )
self.quantities.update(quantities_dynamic)
def _register(self, annotation, prefix, repeats):
diff --git a/nomad/metainfo/metainfo.py b/nomad/metainfo/metainfo.py
index 3c155c8d6a..d047228ea5 100644
--- a/nomad/metainfo/metainfo.py
+++ b/nomad/metainfo/metainfo.py
@@ -27,13 +27,9 @@ import warnings
from collections.abc import Iterable
from copy import deepcopy
from functools import wraps
-from typing import (
- Any,
- TypeVar,
- cast,
- Literal,
-)
-from collections.abc import Callable as TypingCallable
+from typing import Any
+from typing import Callable as TypingCallable
+from typing import Literal, TypeVar, cast
from urllib.parse import urlsplit, urlunsplit
import docstring_parser
@@ -43,26 +39,20 @@ from pydantic import TypeAdapter, ValidationError
from typing_extensions import deprecated # type: ignore
from nomad.config import config
-from nomad.metainfo.data_type import (
- Datatype,
- normalize_type,
- Number,
- m_str,
- Enum,
- Datetime as DatetimeType,
- Unit as UnitType,
- Capitalized as CapitalizedType,
- JSON as JSONType,
- Bytes as BytesType,
- Callable as CallableType,
- URL as URLType,
- Dimension as DimensionType,
- File as FileType,
- Any as AnyType,
- check_dimensionality,
- ExactNumber,
- InexactNumber,
-)
+from nomad.metainfo.data_type import JSON as JSONType
+from nomad.metainfo.data_type import URL as URLType
+from nomad.metainfo.data_type import Any as AnyType
+from nomad.metainfo.data_type import Bytes as BytesType
+from nomad.metainfo.data_type import Callable as CallableType
+from nomad.metainfo.data_type import Capitalized as CapitalizedType
+from nomad.metainfo.data_type import Datatype
+from nomad.metainfo.data_type import Datetime as DatetimeType
+from nomad.metainfo.data_type import Dimension as DimensionType
+from nomad.metainfo.data_type import Enum, ExactNumber
+from nomad.metainfo.data_type import File as FileType
+from nomad.metainfo.data_type import InexactNumber, Number
+from nomad.metainfo.data_type import Unit as UnitType
+from nomad.metainfo.data_type import check_dimensionality, m_str, normalize_type
from nomad.metainfo.util import (
MQuantity,
MSubSectionList,
@@ -74,11 +64,14 @@ from nomad.metainfo.util import (
to_dict,
)
from nomad.units import ureg as units
+from pydantic import ValidationError, parse_obj_as
+from typing_extensions import deprecated # type: ignore
+
from .annotation import (
Annotation,
+ AnnotationModel,
DefinitionAnnotation,
SectionAnnotation,
- AnnotationModel,
)
# todo: remove once simulation package does not use it anymore
@@ -1927,7 +1920,7 @@ class MSection(metaclass=MObjectMeta):
target_value = quantity.default
def _transform_wrapper(_value, _stack=None):
- _path = path
+ _path = path if path else ''
if _stack is not None:
_path += '/' + '/'.join(str(i) for i in _stack)
return (
diff --git a/nomad/search.py b/nomad/search.py
index 329bfe1a09..eb34d65764 100644
--- a/nomad/search.py
+++ b/nomad/search.py
@@ -514,7 +514,7 @@ def _es_to_entry_dict(
quantity = doc_type.quantities.get(id)
if not quantity:
path, schema, _ = parse_quantity_name(id)
- if schema and schema.startswith(yaml_prefix):
+ if schema and schema.startswith((yaml_prefix, nexus_prefix)):
dtype_map = {
'float_value': float,
'str_value': str,
@@ -530,9 +530,6 @@ def _es_to_entry_dict(
quantity = get_quantity(
Quantity(type=dtype), path, schema, doc_type
)
- elif path.startswith(nexus_prefix):
- definition = get_definition(path)
- quantity = get_quantity(definition, path, schema, doc_type)
else:
continue
value_field_name = get_searchable_quantity_value_field(
@@ -699,9 +696,9 @@ def validate_quantity(
if quantity is None:
path, schema, dtype = parse_quantity_name(quantity_name)
- # Queries targeting YAML are translated into dynamic quantity searches
- # on the fly using the provided data type.
- if schema and schema.startswith(yaml_prefix):
+ # Queries targeting YAML or nexus schema are translated into dynamic
+ # quantity searches on the fly using the provided data type.
+ if schema and schema.startswith((yaml_prefix, nexus_prefix)):
datatype = {
'int': int,
'str': str,
@@ -713,23 +710,12 @@ def validate_quantity(
raise QueryValidationError(
(
f'Could not resolve the data type for quantity {quantity_name}. '
- 'Please include the data type in the quantity name for quantities '
- 'that target a custom YAML schema.'
+ 'Please include the data type in the quantity name using the '
+ '"#<type>" postfix.'
),
loc=[quantity_name] if loc is None else loc,
)
quantity = get_quantity(Quantity(type=datatype), path, schema, doc_type)
- # Queries targeting nexus are translated into dynamic quantity searches
- # on the fly by looking at the definition in the metainfo.
- elif path.startswith(nexus_prefix):
- try:
- definition = get_definition(path)
- quantity = get_quantity(definition, path, schema, doc_type)
- except Exception as e:
- raise QueryValidationError(
- f'Could not find the definition for "{path}" in the metainfo.',
- loc=[quantity_name] if loc is None else loc,
- ) from e
else:
raise QueryValidationError(
f'{quantity_name} is not a {doc_type} quantity',
diff --git a/tests/fixtures/data.py b/tests/fixtures/data.py
index 350fa8e6d7..190bdde295 100644
--- a/tests/fixtures/data.py
+++ b/tests/fixtures/data.py
@@ -488,44 +488,6 @@ def example_data_schema_python(
data.delete()
-@pytest.fixture(scope='function')
-def example_data_nexus(
- elastic_module, raw_files_module, mongo_module, user1, normalized
-):
- """
- Contains entries that store data using a python schema.
- """
- data = ExampleData(main_author=user1)
- upload_id = 'id_nexus_published'
-
- data.create_upload(upload_id=upload_id, upload_name=upload_id, published=True)
- data.create_entry(
- upload_id=upload_id,
- entry_id=f'test_entry_nexus',
- mainfile=f'test_content/test.archive.json',
- search_quantities=[
- SearchableQuantity(
- id=f'nexus.NXiv_temp.ENTRY.DATA.temperature__field',
- definition='nexus.NXiv_temp.ENTRY.DATA.temperature__field',
- path_archive='nexus.NXiv_temp.ENTRY.0.DATA.0.temperature__field',
- float_value=273.15,
- ),
- SearchableQuantity(
- id=f'nexus.NXiv_temp.ENTRY.definition__field',
- definition='nexus.NXiv_temp.NXentry.definition__field',
- path_archive='nexus.NXiv_temp.ENTRY.0.definition__field',
- str_value='NXiv_temp',
- ),
- ],
- )
- data.save(with_files=False)
-
- yield
-
- # The data is deleted after fixture goes out of scope
- data.delete()
-
-
@pytest.fixture(scope='function')
def example_data_schema_yaml(
elastic_module,
--
GitLab