diff --git a/docs/howto/customization/hdf5.md b/docs/howto/customization/hdf5.md index 3b37d324884ec7ea3fe202eebb77f43bf96c573b..4553a421144d0082b03f515c663bf583e9d3aeb5 100644 --- a/docs/howto/customization/hdf5.md +++ b/docs/howto/customization/hdf5.md @@ -139,14 +139,13 @@ with deserialized.data.value as dataset: NOMAD clients (e.g. NOMAD UI) can pick up on these HDF5 serialized quantities and provide respective functionality (e.g. showing a H5Web view). +The [H5WebAnnotation class](../../reference/annotations#h5web) contains the attributes that NOMAD will include in the HDF5 file. + <figure markdown>  <figcaption>Visualizing archive HDF5 reference quantity using H5Web.</figcaption> </figure> -!!! warning "Attention" - - This part of the documentation is still work in progress. ## Metadata for large quantities diff --git a/docs/reference/annotations.md b/docs/reference/annotations.md index 621fa7dd75f64b136bf6c9205902428f42884434..9aee5fb07721d4a1fd58335904ca223d71ab1759 100644 --- a/docs/reference/annotations.md +++ b/docs/reference/annotations.md @@ -281,3 +281,78 @@ class CustomSection(PlotSection, EntryData): ``` {{ pydantic_model('nomad.datamodel.metainfo.annotations.PlotAnnotation', heading='### PlotAnnotation (Deprecated)') }} + +## H5Web +The H5WebAnnotation provides a way to control how H5Web renders visualization for +[HDF5Dataset](../howto/customization/hdf5.html#hdf5dataset) quantities. The annotation values are written to the +corresponding HDF5 object attributes and are subsequently read by H5Web. + +Usage: + +- Use this base section as a type in your Quantity. +- Add addictional annotations to trigger the H5Web visualizer to the section containing the Quantity, and, optionally, to its parent sections. + +!!! note + If an EntryData class contain the `a_h5web` annotation, the H5Web plot is shown in the [entry overview page](../examples/computational_data/uploading.html#entries-overview-page). + +An example of use of H5WebAnnotation in Python schemas: + +```python +from nomad.datamodel.data import ArchiveSection, EntryData +from nomad.metainfo import Section +from nomad.datamodel.hdf5 import HDF5Dataset +from nomad.datamodel.metainfo.annotations import H5WebAnnotation + +class A(ArchiveSection): + + m_def = Section(a_h5web=H5WebAnnotation( + axes='x', + signal='y' + auxiliary_signals=['y_err'])) + + x = Quantity( + type=HDF5Dataset, + unit='s', + a_h5web = H5WebAnnotation( + long_name='my_x_label (s)' + )) + + y = Quantity( + type=HDF5Dataset, + unit='m', + a_h5web = H5WebAnnotation( + long_name='my_y_label (m)' + )) + + y_err = Quantity( + type=HDF5Dataset, + unit='m', + h5web = H5WebAnnotation( + long_name='my_y_err_label' + )) + +class B(EntryData): + + m_def = Section(a_h5web=H5WebAnnotation( + axes='value_x', signal='value_y', paths=['a/0'])) + + a = SubSection(sub_section=A, repeats=True) + + value_x = Quantity(type=HDF5Dataset) + + value_y = Quantity(type=HDF5Dataset, unit='m') +``` + +In this example, an H5Web view of the variables `x`, `y`, and `y_err` is displayed in the page of the subsection `A`. +The plot of variables `x_value` and `y_value` is also displayed; as the `B` class is an `EntryData`, the plot is shown in the [entry overview page](../examples/computational_data/uploading.html#entries-overview-page). Additionally, alongside with the plot of the class `B`, the overview page presents also the plot contained in the subsection `A`, due to the `paths` attribute. The `paths` attribute allows to show in a parent section or subsection the plots originally contained in children subsections. Parents and children refer here to **composed** object. + +!!! note + The `paths` variable points in the example above to a [repeated subsection](../howto/plugins/schema_packages.html#schemapackage-class), hence the path provided includes a serial number pointing to the subsection object to be displayed in the entry overview page. To show in the overview page a non-repeatable subsection, no serial number is required in the path. + +H5Web implements visualization features through the attributes shown above, they can be attached to datasets and groups of an HDF5 file. +The conventions for the attributes are rooted in the NeXus language and more explanations can be found in the [NXData documentation page](https://manual.nexusformat.org/classes/base_classes/NXdata.html) and in the [Associating plottable data documentation page](https://manual.nexusformat.org/datarules.html#design-findplottable-niac2014). + +### H5WebAnnotation + +{{ pydantic_model('nomad.datamodel.metainfo.annotations.H5WebAnnotation', heading = '') }} + diff --git a/gui/src/components/archive/ArchiveBrowser.js b/gui/src/components/archive/ArchiveBrowser.js index e4e2309c97af1f1ee5c5f20f96935cc26864662d..f59e3a31faae4a06414f54b1a2c2eca2285e6f36 100644 --- a/gui/src/components/archive/ArchiveBrowser.js +++ b/gui/src/components/archive/ArchiveBrowser.js @@ -762,6 +762,16 @@ QuantityItemPreview.propTypes = ({ def: PropTypes.object.isRequired }) +const matchH5Path = (path) => { + const h5Path = path.match(/(?:\/uploads\/(?<uploadId>.+?)\/(?<source>.+?)\/)*(?<filename>.+?)#(?<path>.+)/) + if (!h5Path) { + return {} + } + const h5File = h5Path.groups.filename + const source = h5Path.groups.source || ((h5File.endsWith('.h5') || h5File.endsWith('.nxs')) ? 'raw' : 'archive') + return {h5UploadId: h5Path.groups.uploadId, h5File: h5File, h5Source: source, h5Path: h5Path.groups.path} +} + export const QuantityValue = React.memo(function QuantityValue({value, def}) { const {uploadId} = useEntryStore() || {} const displayUnit = useDisplayUnit(def) @@ -836,12 +846,9 @@ export const QuantityValue = React.memo(function QuantityValue({value, def}) { })} </ul> } else if (def.type?.type_data === 'nomad.datamodel.hdf5.HDF5Dataset' || def.type?.type_data === 'nomad.datamodel.hdf5.HDF5Reference') { - const h5Path = value.match(/(?:\/uploads\/(?<uploadId>.+?)\/(?<source>.+?)\/)*(?<filename>.+?)#(?<path>.+)/) - const h5UploadId = h5Path.groups.uploadId || uploadId - const h5File = h5Path.groups.filename - const h5Source = h5Path.groups.source || ((h5File.endsWith('.h5') || h5File.endsWith('.nxs')) ? 'raw' : 'archive') + const {h5UploadId, h5File, h5Source, h5Path} = matchH5Path(value) return <Compartment title='hdf5'> - <H5Web upload_id={h5UploadId} filename={h5File} initialPath={h5Path.groups.path} source={h5Source} sidebarOpen={false}></H5Web> + <H5Web upload_id={h5UploadId || uploadId} filename={h5File} initialPath={h5Path} source={h5Source} sidebarOpen={false}></H5Web> </Compartment> } else if (def?.type?.type_kind === 'custom' && def?.type?.type_data === 'nomad.datamodel.data.Query') { return <Query value={value} def={def}/> @@ -944,6 +951,49 @@ export function getAllVisibleProperties(sectionDef) { return [...quantities, ...sub_sections] } +export function H5WebView({section, def, uploadId, title}) { + const h5Web = (section, def) => { + const signal = def.m_annotations?.h5web?.[0]?.signal + if (!signal || !section[signal]) { + return + } + const {h5UploadId, h5File, h5Source, h5Path} = matchH5Path(section[signal]) + const sectionPath = h5Path.split('/').slice(0, -1).join('/') + return <H5Web key={def.name} upload_id={h5UploadId || uploadId} filename={h5File} initialPath={sectionPath} source={h5Source} sidebarOpen={false}></H5Web> + } + const resolve = (path, parent) => { + let child = parent + for (const segment of path.split('/')) { + const properties = child?._properties || child + if (!properties) { + return + } + const index = parseInt(segment) + child = isNaN(index) ? properties[segment] : properties[index] || child + child = child?.sub_section || child + } + return child + } + + const paths = def.m_annotations?.h5web?.[0].paths || [] + + return <Compartment title={title}> + {h5Web(section, def)} + {paths.map(path => { + const subSection = resolve(path, section) + const subSectionDef = resolve(path, def) + return subSection && subSectionDef && h5Web(subSection, subSectionDef) + })} + </Compartment> +} + +H5WebView.propTypes = ({ + section: PropTypes.object.isRequired, + def: PropTypes.object.isRequired, + uploadId: PropTypes.string.isRequired, + title: PropTypes.string +}) + export function Section({section, def, property, parentRelation, sectionIsEditable, sectionIsInEln}) { const {handleArchiveChanged, uploadId, entryId} = useEntryStore() || {} const config = useRecoilValue(configState) @@ -1120,6 +1170,7 @@ export function Section({section, def, property, parentRelation, sectionIsEditab )} {subSectionCompartment} {(def.m_annotations?.plot || def._allBaseSections.map(section => section.name).includes('PlotSection')) && <SectionPlots sectionDef={def} section={section} uploadId={uploadId} entryId={entryId}/>} + {def.m_annotations?.h5web && <H5WebView section={section} def={def} uploadId={uploadId} title='hdf5'></H5WebView>} </React.Fragment> } else { const attributes = section?.m_attributes || {} @@ -1138,6 +1189,7 @@ export function Section({section, def, property, parentRelation, sectionIsEditab ))} </Compartment>} {(def.m_annotations?.plot || def._allBaseSections.map(section => section.name).includes('PlotSection')) && <SectionPlots sectionDef={def} section={section} uploadId={uploadId} entryId={entryId}/>} + {def.m_annotations?.h5web && <H5WebView section={section} def={def} uploadId={uploadId} title='hdf5'></H5WebView>} </React.Fragment> } const eln = def?.m_annotations?.eln diff --git a/gui/src/components/entry/OverviewView.js b/gui/src/components/entry/OverviewView.js index 2d80f11d71a72f4ffb902d712870d64c52928119..447db48239ab1d0511dd99cbdf0e789a5e0fb544 100644 --- a/gui/src/components/entry/OverviewView.js +++ b/gui/src/components/entry/OverviewView.js @@ -52,6 +52,7 @@ import ReferenceUsingCard from "./properties/ReferenceCard" import SampleHistoryUsingCard from "./properties/SampleHistoryCard" import { useEntryStore, useEntryContext, useIndex } from './EntryContext' import DeleteEntriesButton from '../uploads/DeleteEntriesButton' +import HDF5DatasetCard from './properties/HDF5DatasetCard' function MetadataSection({title, children}) { return <Box marginTop={2} marginBottom={2}> @@ -190,6 +191,7 @@ const OverviewView = React.memo(() => { const cardMap = { definitions: DefinitionsCard, nexus: NexusCard, + hdf5dataset: HDF5DatasetCard, material: index?.results?.material?.topology ? MaterialCardTopology : index?.results?.properties?.structures diff --git a/gui/src/components/entry/properties/HDF5DatasetCard.js b/gui/src/components/entry/properties/HDF5DatasetCard.js new file mode 100644 index 0000000000000000000000000000000000000000..c5a2f86950b1b7f9411488959b7e81b463866486 --- /dev/null +++ b/gui/src/components/entry/properties/HDF5DatasetCard.js @@ -0,0 +1,48 @@ +/* eslint-disable quotes */ +/* + * Copyright The NOMAD Authors. + * + * This file is part of NOMAD. See https://nomad-lab.eu for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react' +import PropTypes from 'prop-types' +import { resolveNomadUrlNoThrow } from '../../../utils' +import { useEntryStore } from '../EntryContext' +import { useMetainfoDef } from '../../archive/metainfo' +import { H5WebView } from '../../archive/ArchiveBrowser' +import { PropertyCard } from './PropertyCard' + +const HDF5DatasetCard = React.memo(function HDF5DatasetCard({archive}) { + const {url, uploadId} = useEntryStore() + const m_def = archive?.data?.m_def_id ? `${archive.data.m_def}@${archive.data.m_def_id}` : archive?.data?.m_def + const dataMetainfoDefUrl = url && resolveNomadUrlNoThrow(m_def, url) + const dataMetainfoDef = useMetainfoDef(dataMetainfoDefUrl) + + if (!dataMetainfoDef || !dataMetainfoDef.m_annotations?.h5web) { + return null + } + + return ( + <PropertyCard title='HDF5 Data'> + <H5WebView section={archive.data} def={dataMetainfoDef} uploadId={uploadId}></H5WebView> + </PropertyCard> + ) +}) +HDF5DatasetCard.propTypes = { + archive: PropTypes.object.isRequired +} + +export default HDF5DatasetCard diff --git a/gui/tests/env.js b/gui/tests/env.js index d0a3b27971a12942ac634aec1d19d9c711e545a6..3420ae8a91c2ccffa24aa1fd01de68013c8df4b0 100644 --- a/gui/tests/env.js +++ b/gui/tests/env.js @@ -377,6 +377,9 @@ window.nomadEnv = { "nexus": { "error": "Could not render NeXus card." }, + "hdf5dataset": { + "error": "Could not render HDF5Dataset card." + }, "material": { "error": "Could not render material card." }, diff --git a/nomad/config/defaults.yaml b/nomad/config/defaults.yaml index 53820b6b7fe4e20e0430b8236fdaa893ea74119d..d248f503040770d34e0798f06c82a502c80017a5 100644 --- a/nomad/config/defaults.yaml +++ b/nomad/config/defaults.yaml @@ -510,6 +510,8 @@ ui: error: Could not render definitions card. nexus: error: Could not render NeXus card. + hdf5dataset: + error: Could not render HDF5Dataset card. material: error: Could not render material card. solarcell: diff --git a/nomad/datamodel/hdf5.py b/nomad/datamodel/hdf5.py index b2aa966d6025ccef04e195db2ede2705ec429997..b32422ecd0612331cb6044e6b120b8ee5c4b836f 100644 --- a/nomad/datamodel/hdf5.py +++ b/nomad/datamodel/hdf5.py @@ -26,6 +26,7 @@ import pint from h5py import File from nomad.metainfo.data_type import NonPrimitive +from nomad.datamodel.metainfo.annotations import H5WebAnnotation from nomad.utils import get_logger LOGGER = get_logger(__name__) @@ -156,19 +157,49 @@ class HDF5Dataset(NonPrimitive): else: segment = path else: + unit = self._definition.unit if isinstance(value, pint.Quantity): - if self._definition.unit is not None: - value = value.to(self._definition.unit).magnitude + if unit is not None: + value = value.to(unit).magnitude else: + unit = value.units value = value.magnitude + segment = f'{section.m_path()}/{self._definition.name}' with File(hdf5_path, 'a') as hdf5_file: - segment = f'{section.m_path()}/{self._definition.name}' - target_dataset = hdf5_file.require_dataset( - segment, + target_group = hdf5_file.require_group(section.m_path()) + target_dataset = target_group.require_dataset( + self._definition.name, shape=getattr(value, 'shape', ()), dtype=getattr(value, 'dtype', None), ) target_dataset[...] = value + # add attrs + if unit is not None: + unit = format(unit, '~') + target_dataset.attrs['units'] = unit + + annotation_key = 'h5web' + annotation: H5WebAnnotation = section.m_def.m_get_annotation( + annotation_key + ) + if not annotation: + annotation = section.m_get_annotation(annotation_key) + if annotation: + target_group.attrs.update( + { + key: val + for key, val in annotation.dict().items() + if val is not None + } + ) + target_group.attrs['NX_class'] = 'NXdata' + + q_annotation = self._definition.m_get_annotation(annotation_key) + long_name = q_annotation.long_name if q_annotation else None + if long_name is None: + long_name = self._definition.name + long_name = f'{long_name} ({unit})' if unit else long_name + target_dataset.attrs['long_name'] = long_name return HDF5Wrapper(hdf5_path, segment) diff --git a/nomad/datamodel/metainfo/annotations.py b/nomad/datamodel/metainfo/annotations.py index c5ae113ece24cda391eaf0c36172922f65efb882..39b1c5a1b0252d3dc21818905ec47aded28c1253 100644 --- a/nomad/datamodel/metainfo/annotations.py +++ b/nomad/datamodel/metainfo/annotations.py @@ -983,9 +983,47 @@ class PlotAnnotation(AnnotationModel): return value +class H5WebAnnotation(AnnotationModel): + """ + Provides interface to the H5Web visualizer for HDF5 files. + The annotation can be used for section and quantity definitions in order to + include group and dataset attributes to the archive HDF5 file. + + Refer to https://h5web.panosc.eu/ for a more detailed description + of the annotation fields. + """ + + axes: Union[str, List[str]] = Field( + None, + description=""" + Names of the HDF5Dataset quantities to plot on the independent axes. + """, + ) + signal: str = Field( + None, + description=""" + Name of the HDF5Dataset quantity to plot on the dependent axis. + """, + ) + long_name: str = Field( + None, + description=""" + Label for the hdf5 dataset. Note: this attribute will overwrite also the unit. + """, + ) + auxiliary_signals: List[str] = Field( + None, + description=""" + Additional datasets to include in plot as signal. + """, + ) + paths: List[str] = Field([], description="""List of section paths to visualize.""") + + AnnotationModel.m_registry['eln'] = ELNAnnotation AnnotationModel.m_registry['browser'] = BrowserAnnotation AnnotationModel.m_registry['tabular_parser'] = TabularParserAnnotation AnnotationModel.m_registry['tabular'] = TabularAnnotation AnnotationModel.m_registry['hdf5'] = HDF5Annotation AnnotationModel.m_registry['plot'] = PlotAnnotation +AnnotationModel.m_registry['h5web'] = H5WebAnnotation