diff --git a/dependencies/nomad-remote-tools-hub b/dependencies/nomad-remote-tools-hub index df6aa87b2dc54dc1626e653a0b18f255c6f17561..bc43549f637dccb1aa190f8440b8250d52769100 160000 --- a/dependencies/nomad-remote-tools-hub +++ b/dependencies/nomad-remote-tools-hub @@ -1 +1 @@ -Subproject commit df6aa87b2dc54dc1626e653a0b18f255c6f17561 +Subproject commit bc43549f637dccb1aa190f8440b8250d52769100 diff --git a/dependencies/parsers/nexus b/dependencies/parsers/nexus index 78b721318cb824e404234702c0aef19a7d8b6dba..5c853c9a0e4e7aceaa11e68df59fa3b5e280b7d3 160000 --- a/dependencies/parsers/nexus +++ b/dependencies/parsers/nexus @@ -1 +1 @@ -Subproject commit 78b721318cb824e404234702c0aef19a7d8b6dba +Subproject commit 5c853c9a0e4e7aceaa11e68df59fa3b5e280b7d3 diff --git a/examples/data/apm/README.md b/examples/data/apm/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8bc0b8be583aad945c6c2f9253c1f99d0da90a2a --- /dev/null +++ b/examples/data/apm/README.md @@ -0,0 +1,105 @@ +# An Electronic Lab Notebook (ELN) Example for Atom Probe Microscopy (APM) + +## Introduction + +This example shows how the NOMAD ELN functionalities can be used to collect +required metadata for groups, fields, and attributes for creating datasets +which are compliant with the NeXus NXapm application definition. + +The NeXus NXapm data model is documented here [NXapm](https://manual.nexusformat.org/classes/contributed_definitions/NXapm.html#nxapm) + +Specifically, this example works together with vendor/community file formats +(POS, ePOS, APT, RNG, RRNG) which contain tomographic reconstruction data with +associated ranging data to interpret ion species from mass-to-charge-state ratio +values. This ELN can be used to collect metadata which are not stored in +vendor or community files, like details about the specimen, the instrument, +the users, and post-processing steps. + +This example upload contains the following entries +- A schema in NOMAD's *archive.yaml* format: *nxapm.schema.archive.yaml* +- A schema instance file used by NOMAD *nxapm.archive.json* filled for educational purpose with values for the example. +- The primary consumer of this json file is NOMAD and its internal data management system. +- Another schema instance file used by the nomad-parser-nexus *eln_data.yaml*. This file contains all entered +quantities from the ELN GUI (after the save button was stored). +This file stores metadata with value and units. The file is updated each time the save button in the ELN GUI is clicked. +The file is used by the [NOMAD-PARSER-NEXUS](https://github.com/nomad-coe/nomad-parser-nexus) which creates +NeXus files, like those for atom probe matching the standardized NXapm application definition. +This example is configured to take an example dataset and call the nomad-parser-nexus dataconverter +which creates a NeXus file compliant with NXapm. Once completed this file is available in the upload section/staging area. +This makes also these files explorable with H5Web visualization through the Files menu of the upload section. + +This examples comes with an example measured dataset which is meant for testing and exploration +of the ELN and NORTH container functionalities here shown in this example. The data inside +eln_data.yaml are purely meant for educational purposes. The scientific dataset is a tomographic +reconstruction of a ODS steel specimen measured and characterized by [J. Wang and coworkers](https://doi.org/10.1017/S1431927618015386) +- R31_06365-v02.pos (the reconstruction) +- R31_06365-v02.rrng (the range file) + +## Creating NeXus files + +When you modify the ELN and click the save button the data from the ELN will be +parsed and combined with the POS and RRNG file to create the NXS file. + +You can use this example as a template to translate your own datasets into NeXus. +For this you can drag-and-drop a reconstruction and range file in the respective +file upload fields of the ELN and edit all fields in the ELN accordingly. +After clicking the save button, the newly entered metadata and files are processed +and a new NeXus file, compliant with NXapm is generated. +Currently the implementation supports reconstructions in POS, ePOS and APT file format +(the one generated by APSuite from AMETEK/Cameca). Range files are accepted in +RNG or RRNG format. + +## Where to go from now + +With an example in your upload **you should go to the Analytics tab in the** +Nomad OASIS menu bar and **start up the apmtools container**. The container connects +your upload with a set of containerized software tools for post-processing and +exploring atom probe data in substantial more detail. + +These tools contain currently the +- [Leoben APT_analyzer](https://github.com/areichm/APT_analyzer) by Alexander Reichmann et al. (Lorenz Romaner's team at Montanuniversität Leoben) +- The [paraprobe-toolbox](https://gitlab.com/paraprobe/paraprobe-toolbox) by Markus Kühbach et al. (developed mainly at his time with the Max-Planck-Institut für Eisenforschung GmbH) + +**When it runs the apmtools container offers a jupyter lab server in which the tools are installed.** +**And further examples are available. To access these tools go into the /home/atom_probe_tools directory.** +You will find further instructions there. + +Once you have explored your dataset with these tools you should also explore and realize that the +containers are mounted with your uploads section. The respective directory in the apmtools container +is the **/config** directory. The key benefit of this container design is that you can use a simple +command in a jupyter notebook cell after your analyses + +``` +! mv *.png /config +``` + +This will move for example all generated figures (replace the file extension with a specific +name or file type) so that the files are from this point onwards available in the upload section. +Again H5Web can be used to display the content of HDF5 files, such as the results and config files +of the paraprobe-toolbox. Please keep in mind that especially the results file of paraprobe-toolbox +are not yet fully described with NeXus, so no default plots are available but you can conveniently +explore all your data. + +Equally so you can create jupyter notebooks which post-process these result and move them in the +same manner into your upload area. + +## Summary + +The schema is meant as a starting point. You can download the schema file and extend the +schema to collect further metadata (e. g. for adding the optional quantities +in NXapm). For your own ELNs, you have to consider the data that you acquire +in your lab and create your own schema based on the concepts shown in this example. + +Consult our [documentation on the NOMAD Archive and Metainfo](https://nomad-lab.eu/prod/v1/docs/archive.html) +to learn more about schemas. + +## Questions, comments, suggestions? + +For general questions regarding the APM tools and if you're interested in building one for your +own research workflow or your colleagues and group you are welcome to contact [Markus Kühbach](https://www.fair-di.eu/fairmat/fairmat_/fairmatteam) from the FAIRmat consortium. + + +## Known bugs + +Due to what is likely a bug in H5Web it may currently not be possible to export +the content of files displayed via H5Web using the tool's CSV/NPY export button. \ No newline at end of file diff --git a/examples/data/apm/eln_data.yaml b/examples/data/apm/eln_data.yaml new file mode 100644 index 0000000000000000000000000000000000000000..806cf04d3bb49fc6216003c54785647dae6d5110 --- /dev/null +++ b/examples/data/apm/eln_data.yaml @@ -0,0 +1,67 @@ +atom_probe: + analysis_chamber_pressure: + unit: torr + value: 1.0e-10 + flight_path_length: + unit: m + value: 0.12 + instrument_name: LEAP 3000 + ion_detector_name: unnamed + ion_detector_type: mcp_dld + ion_impact_positions_detection_rate: 0.6 + # reflectron_applied: false, in the ELN GUI the field was not ticked + # therefore the quantities is not picked up by this eln_data + local_electrode_name: electrode 1 + pulser: + pulse_fraction: 0.1 + pulse_frequency: + unit: kHz + value: 250 + pulse_mode: laser + reflectron_applied: false + specimen_monitoring_initial_radius: + unit: nm + value: 30.000000000000004 + specimen_monitoring_shank_angle: + unit: "\xB0" + value: 5 + stage_lab_base_temperature: + unit: K + value: 30 +control_software: + program: IVAS + program__attr_version: 3.6.4 +entry: + attr_version: NeXus 2022.06 commitID d9574a8f90626a929c677f1505729d1751170989 NXapm + definition: NXapm + experiment_description: some details for nomad, ODS steel precipitates do cluster + analysis + experiment_identifier: R31-06365-v02 + operation_mode: apt + program: IVAS + program__attr_version: 3.6.4 + run_number: '6365' + start_time: '2017-07-03T09:49:00+00:00' +operator: +- email: undisclosed + name: Jing Wang +- email: undisclosed + name: Daniel Schreiber +ranging: + program: IVAS + program__attr_version: 3.6.4 +reconstruction: + parameter: kf = 1.8, ICF = 1.02, Vat = 60 at/nm^3, crystallographic calibration + according to some DOI-referended paper + program: IVAS + program__attr_version: 3.6.4 + protocol_name: cameca +specimen: + atom_types: + - Fe + - Cr + - Y + - O + name: ODS-Specimen 1 + preparation_date: '2017-06-13T09:52:00+00:00' + sample_history: undocumented diff --git a/examples/data/apm/nxapm.archive.json b/examples/data/apm/nxapm.archive.json new file mode 100644 index 0000000000000000000000000000000000000000..01713138330edcc3298f7be7c736386e275b1bd2 --- /dev/null +++ b/examples/data/apm/nxapm.archive.json @@ -0,0 +1,78 @@ +{ + "data": { + "m_def": "../upload/raw/nxapm.schema.archive.yaml#/definitions/section_definitions/0", + "reader": "apm", + "nxdl": "NXapm.nxdl", + "entry": { + "attr_version": "NeXus 2022.06 commitID d9574a8f90626a929c677f1505729d1751170989 NXapm", + "definition": "NXapm", + "experiment_identifier": "R31-06365-v02", + "experiment_description": "some details for nomad, ODS steel precipitates do cluster analysis", + "start_time": "2017-07-03T09:49:00.000Z", + "program": "IVAS", + "program__attr_version": "3.6.4", + "run_number": "6365", + "operation_mode": "apt" + }, + "input_files": [ + "R31_06365-v02.pos", + "R31_06365-v02.rrng", + "eln_data.yaml" + ], + "operator": [ + { + "name": "Jing Wang", + "email": "undisclosed" + }, + { + "name": "Daniel Schreiber", + "email": "undisclosed" + } + ], + "specimen": { + "name": "ODS-Specimen 1", + "sample_history": "undocumented", + "preparation_date": "2017-06-13T09:52:00.000Z", + "atom_types": [ + "Fe", + "Cr", + "Y", + "O" + ] + }, + "atom_probe": { + "pulser": { + "pulse_mode": "laser", + "pulse_frequency": 250, + "pulse_fraction": 0.1 + }, + "instrument_name": "LEAP 3000", + "flight_path_length": 0.12, + "reflectron_applied": false, + "local_electrode_name": "electrode 1", + "ion_detector_type": "mcp_dld", + "ion_detector_name": "unnamed", + "stage_lab_base_temperature": 30, + "analysis_chamber_pressure": 1e-10, + "specimen_monitoring_initial_radius": 30.000000000000005, + "specimen_monitoring_shank_angle": 5, + "ion_impact_positions_detection_rate": 0.6 + }, + "control_software": { + "program": "IVAS", + "program__attr_version": "3.6.4" + }, + "reconstruction": { + "program": "IVAS", + "program__attr_version": "3.6.4", + "protocol_name": "cameca", + "parameter": "kf = 1.8, ICF = 1.02, Vat = 60 at/nm^3, crystallographic calibration according to some DOI-referended paper" + }, + "ranging": { + "program": "IVAS", + "program__attr_version": "3.6.4" + }, + "output": "output.nxs" + }, + "m_ref_archives": {} +} \ No newline at end of file diff --git a/examples/data/apm/nxapm.schema.archive.yaml b/examples/data/apm/nxapm.schema.archive.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d7a82fbb77c432dd4187b5cf7ef1e4091221d7d --- /dev/null +++ b/examples/data/apm/nxapm.schema.archive.yaml @@ -0,0 +1,525 @@ +# group, field, and attribute names match to NXapm, for further details +# what each field should contain consult the respective docstring of the +# quantity in NXapm +definitions: + name: "APM ELN Example" + # 'ELN/application definition schema for atom probe microscopy (APM) experiments.' + sections: # section definitions what were back in the old days msection base classes + # Operator: + # Specimen: + AtomProbeMicroscopy: # the actual schema + # nomad.datamodel.metainfo.eln.NexusParser + base_sections: + - 'nomad.datamodel.metainfo.eln.NexusParser' + - 'nomad.datamodel.data.EntryData' + # base_section: nomad.datamodel.data.EntryData + m_annotations: + # Here you can set your default values for the reader and nxdl. + template: + reader: apm + nxdl: NXapm.nxdl + # Listing quantities in the hide component will not show them in the ELN. + # This would be useful to make the default values set in `template` fixed. + # Leave the hide key even if you want to pass an empty list like in this example. + eln: + # hide: ['nxdl', 'reader'] + hide: [] + sub_sections: + entry: + section: + description: | + Generic details about an experiment. + m_annotations: + eln: + overview: true + quantities: + attr_version: + type: + type_kind: Enum + type_data: + - 'NeXus 2022.06 commitID d9574a8f90626a929c677f1505729d1751170989 NXapm' + description: Hashvalue of the NeXus application definition file + m_annotations: + eln: + component: RadioEnumEditQuantity + definition: + type: + type_kind: Enum + type_data: + - NXapm + description: NeXus NXDL schema to which this file conforms + m_annotations: + eln: + component: RadioEnumEditQuantity + experiment_identifier: + type: str + description: GUID of the experiment + m_annotations: + eln: + component: StringEditQuantity + experiment_description: + type: str + description: Free text details about the experiment + m_annotations: + eln: + component: StringEditQuantity + start_time: + type: Datetime + description: ISO 8601 time code with local time zone offset to UTC when the experiment started. + m_annotations: + eln: + component: DateTimeEditQuantity + # end_time: + # type: Datetime + # description: ISO 8601 time code with local time zone offset to UTC when the experiment ended. + # m_annotations: + # eln: + # component: DateTimeEditQuantity + program: + type: str + description: Name of the program used to create this file. + m_annotations: + eln: + component: StringEditQuantity + program__attr_version: + type: str + description: Version plus build number, commit hash, or description of the program to support reproducibility. + m_annotations: + eln: + component: StringEditQuantity + run_number: + type: str + description: Identifier in the instrument control software given for this experiment. + m_annotations: + eln: + component: StringEditQuantity + # experiment_documentation(NXnote): + # thumbnail(NXnote): + # attr_type: + operation_mode: + type: + type_kind: Enum + type_data: + - apt + - fim + - apt_fim + - other + description: | + What type of atom probe microscope experiment is performed. + APT experiments use no imaging gas while FIM does. + m_annotations: + eln: + component: RadioEnumEditQuantity + # inputfile_reconstruction: + # type: str + # description: | + # Place to drag-and-drop the file containing the result of the measurement. + # This result has to be the tomographic reconstruction. + # Accepted file formats are POS, ePOS, and APT (from APSuite). + # m_annotations: + # eln: + # component: FileEditQuantity + # inputfile_range_file: + # type: str + # description: | + # Place to drag-and-drop a file which contains the result of a ranging of + # the mass-to-charge-state ratio values to assigned ion labels. + # Accepted file formats are RNG, and RRNG. + # m_annotations: + # eln: + # component: FileEditQuantity + operator: + repeats: true + section: + description: | + Contact information and eventually details of at least one person + involved in the taking of the microscope session. + m_annotations: + eln: + quantities: + name: + type: str + description: Given (first) name and surname. + m_annotations: + eln: + component: StringEditQuantity + # affiliation: + # type: str + # description: Name of the affiliation when the experiment was performed. + # m_annotations: + # eln: + # component: StringEditQuantity + # address: + # type: str + # description: Postal address of the affiliation. + # m_annotations: + # eln: + # component: StringEditQuantity + email: + type: str + description: Email address when the experiment was performed. + m_annotations: + eln: + component: StringEditQuantity + # orcid: + # type: str + # description: GUID as offered by services like ORCID or ResearcherID. + # m_annotations: + # eln: + # component: StringEditQuantity + # telephone_number: + # type: str + # description: (Business) (tele)phone number when the experiment was performed. + # m_annotations: + # eln: + # component: StringEditQuantity + # role: + # type: str + # description: | + # Which role does the user have in the place and at the point + # in time when the experiment was performed? Technician operating + # the microscope. Student, postdoc, principle investigator, guest + # are common examples. + # m_annotations: + # eln: + # component: StringEditQuantity + # social_media_name: + # type: str + # description: Account name that is associated with the user on social media platforms. + # m_annotations: + # eln: + # component: StringEditQuantity + # social_media_platform: + # type: str + # description: Name of the social media platform where the account under social_media_name is registered. + # m_annotations: + # eln: + # component: StringEditQuantity + specimen: + section: + description: | + Details about the specimen and its immediate environment. + m_annotations: + eln: + quantities: + name: + type: str + description: | + GUID which distinguishes the specimen from all others and especially + the predecessor/origin from where the specimen was cut. + In cases where the specimen was e.g. site-specifically cut from + samples or in cases of an instrument session during which multiple + specimens are loaded, the name has to be descriptive enough to + resolve which specimen on e.g. the microtip array was taken. + This field must not be used for an alias of the specimen. + Instead, use short_title. + m_annotations: + eln: + component: StringEditQuantity + sample_history: + type: str + description: | + Reference to the location of or a GUID providing as many details + as possible of the material, its microstructure, and its + thermo-chemo-mechanical processing/preparation history. + m_annotations: + eln: + component: StringEditQuantity + preparation_date: + type: Datetime + description: | + ISO 8601 time code with local time zone offset to UTC information when + the measured specimen surface was actively prepared. + m_annotations: + eln: + component: DateTimeEditQuantity + # short_title: + # type: str + # description: Possibility to give an abbreviation of the specimen name field. + # m_annotations: + # eln: + # component: StringEditQuantity + # atom_types should be a list of strings + atom_types: + type: str + shape: ['*'] + description: | + Use Hill's system for listing elements of the periodic table which + are inside or attached to the surface of the specimen and thus + relevant from a scientific point of view. + m_annotations: + eln: + component: StringEditQuantity + # description: + # type: str + # description: | + # Discouraged free text field to be used in the case when properly + # designed records for the sample_history are not available. + # m_annotations: + # eln: + # component: StringEditQuantity + atom_probe: + section: + description: | + The instrument and the lab in which it stands. + m_annotations: + eln: + quantities: + instrument_name: + type: str + description: Given name of the atom probe at the hosting institution. + m_annotations: + eln: + component: StringEditQuantity + # location: + # (NXmanufacturer): + flight_path_length: + type: np.float64 + unit: meter + description: | + The space inside the atom probe that ions pass through nominally + when they leave the specimen and travel to the detector. + m_annotations: + eln: + component: NumberEditQuantity + defaultDisplayUnit: meter + minValue: 0.0 + maxValue: 10.0 + # field_of_view(NX_FLOAT): + # analysis_chamber(NXchamber): + # load_lock_chamber(NXchamber): + # buffer_chamber(NXchamber): + # getter_pump(NXpump): + # roughening_pump(NXpump): + # turbomolecular_pump(NXpump): + # how is it with such an optional group + reflectron_applied: + type: bool + description: Is a reflectron installed and was it used? + m_annotations: + eln: + component: BoolEditQuantity + local_electrode_name: + type: str + description: Identifier of the local_electrode in the control software database. + m_annotations: + eln: + component: StringEditQuantity + # (NXaperture_em): + # ion_detector(NXdetector): + ion_detector_type: + type: + type_kind: Enum + type_data: + - mcp_dld + - phosphor_ccd + - other + description: | + Type of the ToF-taking detector system + Examples are mcp_dld, phosphor_ccd, or other + m_annotations: + eln: + component: RadioEnumEditQuantity + ion_detector_name: + type: str + description: Given name or alias of the detector + m_annotations: + eln: + component: StringEditQuantity + # model: + # serial_number: + # manufacturer_name: + # signal_amplitude(NX_FLOAT): + stage_lab_base_temperature: + type: np.float64 + unit: kelvin + description: | + Average temperature at the specimen base, i.e. + base temperature, during the measurement. + m_annotations: + eln: + component: NumberEditQuantity + defaultDisplayUnit: kelvin + minValue: 0.0 + maxValue: 273.15 + analysis_chamber_pressure: + type: np.float64 + unit: torr + description: | + Average pressure in the analysis chamber during the measurement. + m_annotations: + eln: + component: NumberEditQuantity + defaultDisplayUnit: torr + minValue: 0.0 + maxValue: 1500.12 + specimen_monitoring_initial_radius: + type: np.float64 + unit: nanometer + description: | + Ideally measured or best elaborated guess of the initial radius of the specimen. + m_annotations: + eln: + component: NumberEditQuantity + defaultDisplayUnit: nanometer + minValue: 1.0 + maxValue: 1000.0 + specimen_monitoring_shank_angle: + type: np.float64 + unit: degree + descriptions: | + Ideally measured or best elaborated guess of the shank angle. + This is a measure of the specimen taper. + m_annotations: + eln: + component: NumberEditQuantity + defaultDisplayUnit: degree + minValue: 0.0 + maxValue: 90.0 + ion_impact_positions_detection_rate: + type: np.float64 + descriptions: | + Average detection rate over the course of the experiment. + m_annotations: + eln: + component: NumberEditQuantity + minValue: 0.0 + maxValue: 1.0 + sub_sections: + pulser: + section: + description: Details about the pulsing device and method + m_annotations: + eln: + quantities: + pulse_mode: + type: + type_kind: Enum + type_data: + - laser + - high_voltage + - laser_and_high_voltage + description: | + Which pulsing mode was used? + m_annotations: + eln: + component: RadioEnumEditQuantity + pulse_frequency: + type: np.float64 + unit: kilohertz + description: Pulse frequency + m_annotations: + eln: + component: NumberEditQuantity + defaultDisplayUnit: kilohertz + minValue: 0.0 + maxValue: 10000.0 + pulse_fraction: + type: np.float64 + description: Pulse fraction + m_annotations: + eln: + component: NumberEditQuantity + minValue: 0.0 + maxValue: 1.0 + # pulsed_voltage(NX_FLOAT): + control_software: + section: + description: Which control software was used e.g. IVAS/APSuite + m_annotations: + eln: + quantities: + program: + type: str + description: | + Name of the program used to control the instrument during the measurement. + Examples are IVAS or APSuite for Cameca/AMETEK local electrode atom probes. + m_annotations: + eln: + component: StringEditQuantity + program__attr_version: + type: str + description: Version plus build number, commit hash, or description of the program to support reproducibility. + m_annotations: + eln: + component: StringEditQuantity + reconstruction: + section: + description: Details about the reconstruction + m_annotations: + eln: + quantities: + program: + type: str + description: | + Name of the program used to perform the reconstruction with. + Examples are IVAS, APSuite, or names of open-source tools. + m_annotations: + eln: + component: StringEditQuantity + program__attr_version: + type: str + description: Version plus build number, commit hash, or description of the program to support reproducibility. + m_annotations: + eln: + component: StringEditQuantity + protocol_name: + type: + type_kind: Enum + type_data: + - bas + - geiser + - gault + - cameca + - other + description: | + Qualitative statement about which reconstruction protocol was used. + Bas maps currently to bas_modified. + Cameca maps currently to apsuite. + Add more details in the parameter text field, + e.g. reconstruction parameter. + m_annotations: + eln: + component: RadioEnumEditQuantity + parameter: + type: str + description: | + Different reconstruction protocols exist. Although these approaches + are qualitatively similar, each protocol uses different parameters + (and interprets these differently). The source code to IVAS/APSuite + is not open. For now users should store reconstruction parameter + in a collection, i.e. here with a free-text field. + m_annotations: + eln: + component: StringEditQuantity + # (NXstage_lab): + # (NXdata): + # (NXcoordinate_system_set): + # (NXmonitor): + ranging: + section: + description: Details about the ranging. + m_annotations: + eln: + quantities: + program: + type: str + description: | + Name of the program used to perform the ranging with. + Examples are IVAS, APSuite, or names of open-source tools. + m_annotations: + eln: + component: StringEditQuantity + program__attr_version: + type: str + description: Version plus build number, commit hash, or description of the program to support reproducibility. + m_annotations: + eln: + component: StringEditQuantity + # number_of_iontypes + # maximum_number_of_atoms_per_molecular_ion + # mass_to_charge_distribution + # background_quantification + # peak_search_and_deconvolution + # peak_identification taken over from program and program version + diff --git a/examples/data/generate_example_uploads.sh b/examples/data/generate_example_uploads.sh index 9b981c4539f6c8d766cf6164e18a6a1e5351afc3..604f2bf4bcd14d2677a0604015035dc4af00abd6 100755 --- a/examples/data/generate_example_uploads.sh +++ b/examples/data/generate_example_uploads.sh @@ -1,4 +1,10 @@ #!/bin/sh + +cd ../../ +BASE_PATH=$(pwd) +REMOTE_TOOLS_PATH=$BASE_PATH/dependencies/nomad-remote-tools-hub/docker/ + +cd $BASE_PATH/examples/data rm -rf uploads/*.zip curl -L https://www.dropbox.com/s/8zd7aqe91lza2r4/theory-example-upload.zip?dl=1 -o uploads/theory.zip # TODO this does not work on the mpcdf servers (no route to host) !? @@ -8,4 +14,19 @@ zip -r ../uploads/eln.zip * cd .. cd tabular zip -r ../uploads/tabular.zip * -cd .. \ No newline at end of file +cd ../apm +curl -L https://zenodo.org/record/6808516/files/R31_06365-v02.tar.gz?download=1 -o R31_06365-v02.tar.gz +tar -xvf R31_06365-v02.tar.gz +rm R31_06365-v02.tar.gz +zip -r ../uploads/apm.zip * +cd $REMOTE_TOOLS_PATH/mpes/example +curl -L https://zenodo.org/record/7035754/files/MoTe2.h5?download=1 -o MoTe2.h5 +mkdir -p TiTe2 +cd TiTe2 +curl -L https://zenodo.org/record/5541490/files/TiTe2_0deg.nxs?download=1 -o TiTe2_0deg.nxs +curl -L https://zenodo.org/record/5541490/files/TiTe2_60deg.nxs?download=1 -o TiTe2_60deg.nxs +cd .. +zip -r $BASE_PATH/examples/data/uploads/mpes.zip * +cd $REMOTE_TOOLS_PATH/ellips/example +zip -r $BASE_PATH/examples/data/uploads/ellips.zip * +cd $BASE_PATH/examples/data diff --git a/examples/data/uploads/example_uploads.yml b/examples/data/uploads/example_uploads.yml index 442df6d2e47039193d572fd85bc9ed1fa7a9e3f3..9e039d0893258e64388506fb38bc907463b4e1c4 100644 --- a/examples/data/uploads/example_uploads.yml +++ b/examples/data/uploads/example_uploads.yml @@ -22,3 +22,32 @@ tables: file in combination with a custom schema. The schema describes what the columns in the excel file mean and NOMAD can parse everything accordingly to produce a **FAIR** dataset. +ellips: + path: examples/data/uploads/ellips.zip + title: Ellipsometry + description: | + This example presents the capabilities of the NOMAD platform to store and standardize ellipsometry data. + It shows the generation of a NeXus file according to the [NXellipsometry](https://manual.nexusformat.org/classes/contributed_definitions/NXellipsometry.html#nxellipsometry) + application definition and a successive analysis of a SiO2 on Si Psi/Delta measurement. +mpes: + path: examples/data/uploads/mpes.zip + title: Mpes + description: | + This example presents the capabilities of the NOMAD platform to store and standardize multi photoemission spectroscopy (MPES) experimental data. It contains three major examples: + + - Taking a pre-binned file, here stored in a h5 file, and converting it into the standardized MPES NeXus format. + There exists a [NeXus application definition for MPES](https://manual.nexusformat.org/classes/contributed_definitions/NXmpes.html#nxmpes) which details the internal structure of such a file. + - Binning of raw data (see [here](https://www.nature.com/articles/s41597-020-00769-8) for additional resources) into a h5 file and consecutively generating a NeXus file from it. + - An analysis example using data in the NeXus format and employing the [pyARPES](https://github.com/chstan/arpes) analysis tool to reproduce the main findings of [this paper](https://arxiv.org/pdf/2107.07158.pdf). +apm: + path: examples/data/uploads/apm.zip + title: Electronic Lab Notebook for Atom Probe Microscopy (APM) + description: | + This is an example for atom probe microscopy. + The example contains a custom NOMAD *schema* to create an **Electronic + Lab Notebook (ELN)** with which users can enter metadata that are usually + not stored in vendor or community file formats. The example serves two + purposes. On the one hand it shows how custom NOMAD *schema* can be + created for a research community, here atom probe. On the other hand it + shows how all required data in a NeXus NXapm file can be added to supplement + content from vendor and community files. diff --git a/gui/src/components/archive/FilePreview.js b/gui/src/components/archive/FilePreview.js index 893d479dcb3c11d71bcc00a88aa1aa03c73e09cf..eaffd0cbb470076c03e5f45bcf373f89f82b695b 100644 --- a/gui/src/components/archive/FilePreview.js +++ b/gui/src/components/archive/FilePreview.js @@ -100,7 +100,7 @@ const viewerPDF = { } const viewerHDF5 = { name: 'hdf5', - fileExtensions: ['hdf5', 'hd5', 'nxs'], + fileExtensions: ['hdf5', 'hd5', 'nxs', 'h5', 'nexus', 'nx'], maxSizeAutoPreview: 10e6, width: 'fit-content', render: ({uploadId, path}) => diff --git a/gui/src/components/entry/OverviewView.js b/gui/src/components/entry/OverviewView.js index b1ea5404b87a831f7cede5c25d3fb691ec819b7b..3452d9eb9f04858f1bb5bf26e793e1c74ae56841 100644 --- a/gui/src/components/entry/OverviewView.js +++ b/gui/src/components/entry/OverviewView.js @@ -172,7 +172,7 @@ const OverviewView = React.memo((props) => { return - + @@ -212,7 +212,7 @@ const OverviewView = React.memo((props) => { - + {editable && ( diff --git a/gui/tests/data/search/inputlist.json b/gui/tests/data/search/inputlist.json index 0f4d8b51ba68a5f07c7e34394fa84fc50d3008fe..dc86b6cd84fc57807ce1cad106adebd4ecadfc4f 100644 --- a/gui/tests/data/search/inputlist.json +++ b/gui/tests/data/search/inputlist.json @@ -51,7 +51,7 @@ "page_size": 0, "order_by": "entry_id", "order": "asc", - "total": 3 + "total": 15 }, "required": { "include": [ diff --git a/gui/yarn.lock b/gui/yarn.lock index a6b46b282bc37b98ecb9e7094243030a9f1fc3a3..6a68186a4b15d46a3727f030653ffc5f76d6e90f 100644 --- a/gui/yarn.lock +++ b/gui/yarn.lock @@ -4717,6 +4717,11 @@ clsx@^1.0.2, clsx@^1.0.4, clsx@^1.1.1: resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.2.1.tgz#0ddc4a20a549b59c93a4116bb26f5294ca17dc12" integrity sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg== +clsx@^1.1.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.2.1.tgz#0ddc4a20a549b59c93a4116bb26f5294ca17dc12" + integrity sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg== + co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" @@ -13348,6 +13353,17 @@ react-grid-layout@^1.3.4: react-draggable "^4.0.0" react-resizable "^3.0.4" +react-grid-layout@^1.3.4: + version "1.3.4" + resolved "https://registry.yarnpkg.com/react-grid-layout/-/react-grid-layout-1.3.4.tgz#4fa819be24a1ba9268aa11b82d63afc4762a32ff" + integrity sha512-sB3rNhorW77HUdOjB4JkelZTdJGQKuXLl3gNg+BI8gJkTScspL1myfZzW/EM0dLEn+1eH+xW+wNqk0oIM9o7cw== + dependencies: + clsx "^1.1.1" + lodash.isequal "^4.0.0" + prop-types "^15.8.1" + react-draggable "^4.0.0" + react-resizable "^3.0.4" + react-highlight@^0.14.0: version "0.14.0" resolved "https://registry.yarnpkg.com/react-highlight/-/react-highlight-0.14.0.tgz#5aefa5518baa580f96b68d48129d7a5d2dc0c9ef" diff --git a/nomad/datamodel/metainfo/eln/__init__.py b/nomad/datamodel/metainfo/eln/__init__.py index beefea8fa71f99374492df166f142bd5ce3e54e8..539e3b832c3c344ab98fb402bbe0bea82f117d55 100644 --- a/nomad/datamodel/metainfo/eln/__init__.py +++ b/nomad/datamodel/metainfo/eln/__init__.py @@ -16,14 +16,21 @@ # limitations under the License. # +import re import numpy as np +import yaml + from nomad import utils from nomad.units import ureg from nomad.datamodel.data import EntryData, ArchiveSection from nomad.metainfo.metainfo import SectionProxy from nomad.datamodel.results import ELN, Results, Material, BandGap -from nomad.metainfo import Package, Quantity, Datetime, Reference, Section from nomad.datamodel.metainfo.eln.perovskite_solar_cell_database import addSolarCell +from nomad.metainfo import Package, Quantity, Datetime, Reference, Section, MEnum + +from nexusparser.tools.dataconverter.convert import get_names_of_all_readers, convert +from nexusparser.tools.nexus import get_app_defs_names + m_package = Package(name='material_library') @@ -911,4 +918,65 @@ class SolarCellEQE(ArchiveSection): archive.results.properties.available_properties = props +class NexusParser(ArchiveSection): + + reader = Quantity( + type=MEnum(get_names_of_all_readers()), + description='The reader needed to run the Nexus converter.', + a_eln=dict(component='AutocompleteEditQuantity')) + + nxdl = Quantity( + type=MEnum(get_app_defs_names()), + description='The nxdl needed for running the Nexus converter.', + a_eln=dict(component='AutocompleteEditQuantity')) + + input_files = Quantity( + type=str, + shape=['*'], + description='Input files needed to run the nexus converter.', + a_eln=dict(component='FileEditQuantity'), + a_browser=dict(adaptor='RawFileAdaptor')) + + output = Quantity( + type=str, + description='Output Nexus filename to save all the data. Default: output.nxs', + a_eln=dict(component='StringEditQuantity'), + a_browser=dict(adaptor='RawFileAdaptor'), + default="output.nxs") + + def normalize(self, archive, logger): + super(NexusParser, self).normalize(archive, logger) + + raw_dir = archive.m_context.upload_files._raw_dir + + def transform(quantity_def, section, value, path): + if quantity_def.unit: + return dict(value=value, unit=str(format(quantity_def.unit, '~'))) + return value + + def exclude(quantity_def, section): + return quantity_def.name in ("reader", "input_files", "output", "nxdl") + + eln_dict = archive.m_to_dict(transform=transform, exclude=exclude) + del eln_dict["data"]["m_def"] + with archive.m_context.raw_file("eln_data.yaml", 'w') as eln_file: + yaml.dump(eln_dict["data"], eln_file, allow_unicode=True) + + if archive.data.input_files is None: + archive.data.input_files = [] + + if "eln_data.yaml" not in archive.data.input_files: + archive.data.input_files.append("eln_data.yaml") + + converter_params = {"reader": archive.data.reader, + "nxdl": re.sub(".nxdl$", "", archive.data.nxdl), + "input_file": [str(raw_dir.join_file(file)) + for file in archive.data.input_files], + "output": str(raw_dir.join_file(archive.data.output))} + convert(**converter_params) + + if logger is None: + logger = utils.get_logger(__name__) + + m_package.__init_metainfo__() diff --git a/nomad/jupyterhub_config.py b/nomad/jupyterhub_config.py index c414f6f8b06e50a3debd37c4c0514028c29b4f9a..f04b92f548635d7697ddac720080284ec3aaf18b 100644 --- a/nomad/jupyterhub_config.py +++ b/nomad/jupyterhub_config.py @@ -252,9 +252,26 @@ def create_configure_from_tool_json(tool_json): spawner.cmd = tools_json['cmd'] for key, value in spawner.volumes.items(): - if value.startswith('/prefix') and 'mount_path' in tool_json: + if not isinstance(value, dict) \ + and value.startswith('/prefix') \ + and 'mount_path' in tool_json: spawner.volumes[key] = value.replace('/prefix', tool_json['mount_path']) + mounts = tool_json.get('mounts', {}) + for key, value in mounts.items(): + if isinstance(value, dict) \ + and value['bind'].startswith('/prefix') \ + and 'mount_path' in tool_json: + spawner.volumes[key] = { + 'bind': value['bind'].replace('/prefix', tool_json['mount_path']), + 'mode': value['mode'] + } + + hosts = tool_json.get('extra_hosts', {}) + spawner.extra_host_config.update({ + "extra_hosts": hosts + }) + return configure