From 0f4388cee5f9cc11a3267946392b29920b0ddac9 Mon Sep 17 00:00:00 2001
From: Lauri Himanen <lauri.himanen@gmail.com>
Date: Thu, 22 Aug 2024 07:10:49 +0000
Subject: [PATCH] Resolve "Fix issue with example_uploads.yml not being
included in the python package"
---
docs/howto/plugins/example_uploads.md | 2 +-
examples/data/uploads/.gitkeep | 0
examples/data/uploads/example_uploads.yml | 114 ---------------
gui/src/components/About.js | 10 +-
.../components/uploads/ExampleUploadButton.js | 2 +-
gui/src/config.js | 2 +-
gui/tests/artifacts.js | 73 ----------
gui/tests/env.js | 104 +++++++++++++
nomad/cli/dev.py | 27 ----
nomad/config/defaults.yaml | 137 ++++++++++++++++++
nomad/config/models/config.py | 25 ++--
nomad/config/models/plugins.py | 12 +-
tests/config/models/test_plugins.py | 4 +-
13 files changed, 276 insertions(+), 236 deletions(-)
create mode 100644 examples/data/uploads/.gitkeep
delete mode 100644 examples/data/uploads/example_uploads.yml
diff --git a/docs/howto/plugins/example_uploads.md b/docs/howto/plugins/example_uploads.md
index aa719bb59e..b0d8701b2c 100644
--- a/docs/howto/plugins/example_uploads.md
+++ b/docs/howto/plugins/example_uploads.md
@@ -69,7 +69,7 @@ There are three main ways to include data in an example upload:
myexampleupload = ExampleUploadEntryPoint(
name = 'MyExampleUpload',
description = 'My custom example upload.',
- url='http://my_large_file_address.zip
+ url='http://my_large_file_address.zip'
)
```
diff --git a/examples/data/uploads/.gitkeep b/examples/data/uploads/.gitkeep
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/examples/data/uploads/example_uploads.yml b/examples/data/uploads/example_uploads.yml
deleted file mode 100644
index 3b5a368859..0000000000
--- a/examples/data/uploads/example_uploads.yml
+++ /dev/null
@@ -1,114 +0,0 @@
-Basic examples:
- theory:
- path: examples/data/uploads/theory.zip
- title: Electronic structure code input and output files
- description: |
- This upload demonstrate the basic use of NOMAD's *parsers*. For many *electronic
- structure codes* (VASP, etc.), NOMAD provides parsers. You simply upload
- the *input and output files* of your simulations and NOMAD parsers are extracting
- all necessary metadata to produce a **FAIR** dataset.
- eln:
- path: examples/data/uploads/eln.zip
- title: Electronic Lab Notebook
- description: |
- This example contains a custom NOMAD *schema* to create an **Electronic
- Lab Notebook (ELN)** and a few example *data* entries that use this schema.
- The schema demonstrates the basic concepts behind a NOMAD ELN and can be a good
- **starting point** to create you own schemas that model **FAIR data** acquired in your lab.
- tables:
- path: examples/data/uploads/tabular.zip
- title: Tabular Data
- description: |
- This upload demonstrates the used of tabular data. In this example we use an *xlsx*
- file in combination with a custom schema. The schema describes what the columns
- in the excel file mean and NOMAD can parse everything accordingly to
- produce a **FAIR** dataset.
-Tutorials:
- rdm_tutorial:
- path: examples/data/uploads/rdm_tutorial.zip
- title: Tailored Research Data Management (RDM) with NOMAD
- description: |
- This notebook will teach you how you can build tailored research data
- management (RDM) solutions using NOMAD. It uses existing thermally
- activated delayed fluorescent (TADF) molecule data to teach you how we
- can use the NOMAD to turn research data into datasets that are FAIR:
- Findable, Accessible, Interoperable and Reusable. The end-result can be
- distributed as a NOMAD plugin: a self-contained Python package that can be
- installed on a NOMAD Oasis.
- cow_tutorial:
- path: examples/data/uploads/cow_tutorial.zip
- title: NOMAD as a Data Management Framework Tutorial
- description: |
- This upload provides a notebook as a tutorial that demonstrates how to use NOMAD
- for managing custom data and file types. Based on a simple *Countries of the World*
- dataset, it shows how to model the data in a schema, do parsing and normalization,
- process data, access existing data with NOMAD's API for analysis, and how to
- add visualization to your data.
-FAIRmat examples:
- ellips:
- path: examples/data/uploads/ellips.zip
- title: Ellipsometry
- description: |
- This example presents the capabilities of the NOMAD platform to store and standardize ellipsometry data.
- It shows the generation of a NeXus file according to the [NXellipsometry](https://manual.nexusformat.org/classes/contributed_definitions/NXellipsometry.html#nxellipsometry)
- application definition and a successive analysis of a SiO2 on Si Psi/Delta measurement.
- mpes:
- path: examples/data/uploads/mpes.zip
- title: Mpes
- description: |
- This example presents the capabilities of the NOMAD platform to store and standardize multi photoemission spectroscopy (MPES) experimental data. It contains three major examples:
-
- - Taking a pre-binned file, here stored in a h5 file, and converting it into the standardized MPES NeXus format.
- There exists a [NeXus application definition for MPES](https://manual.nexusformat.org/classes/contributed_definitions/NXmpes.html#nxmpes) which details the internal structure of such a file.
- - Binning of raw data (see [here](https://www.nature.com/articles/s41597-020-00769-8) for additional resources) into a h5 file and consecutively generating a NeXus file from it.
- - An analysis example using data in the NeXus format and employing the [pyARPES](https://github.com/chstan/arpes) analysis tool to reproduce the main findings of [this paper](https://arxiv.org/pdf/2107.07158.pdf).
- xps:
- path: examples/data/uploads/xps.zip
- title: XPS
- description: |
- This example presents the capabilities of the NOMAD platform to store and standardize XPS data.
- It shows the generation of a NeXus file according to the
- [NXmpes](https://manual.nexusformat.org/classes/contributed_definitions/NXmpes.html#nxmpes)
- application definition and a successive analysis of an example data set.
- sts:
- path: examples/data/uploads/sts.zip
- title: STS
- description: |
- This example is for two types of experiments: Scanning Tunneling Microscopy (STM) and Scanning Tunneling Spectroscopy (STS) from Scanning Probe Microscopy.
- It can transform the data from files generated by a the nanonis software into the NeXus application definition NXsts.
- The example contains data files from the two specific nanonis software versions generic 5e and generic 4.5.
- stm:
- path: examples/data/uploads/stm.zip
- title: STM
- description: |
- This example is for two types of experiments: Scanning Tunneling Microscopy (STM) and Scanning Tunneling Spectroscopy (STS) from Scanning Probe Microscopy.
- It can transform the data from files generated by a the nanonis software into the NeXus application definition NXsts.
- The example contains data files from the two specific nanonis software versions generic 5e and generic 4.5.
- apm:
- path: examples/data/uploads/apm.zip
- title: Atom Probe Microscopy
- description: |
- This example presents the capabilities of the NOMAD platform to store and standardize atom probe data.
- It shows the generation of a NeXus file according to the
- [NXapm](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXapm.html#nxapm)
- application definition and a successive analysis of an example data set.
- The example contains a small atom probe dataset from an experiment with a LEAP instrument to get you started
- and keep the size of your NOMAD installation small. Ones started, we recommend to change the respective
- input file in the NOMAD Oasis ELN to run the example with your own datasets.
- em:
- path: examples/data/uploads/em.zip
- title: Electron Microscopy
- description: |
- This example presents the capabilities of the NOMAD platform to store and standardize electron microscopy.
- It shows the generation of a NeXus file according to the
- [NXem](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXem.html#nxem)
- application definition.
- The example contains a small set of electron microscopy datasets to get started and keep the size of your
- NOMAD installation small. Ones started, we recommend to change the respective input file in the NOMAD Oasis
- ELN to run the example with your own datasets.
- iv_temp:
- path: examples/data/uploads/iv_temp.zip
- title: Sensor Scan - IV Temperature Curve
- description: |
- This example shows users how to take data from a Python framework and map it out to a Nexus application definition for IV Temperature measurements, [NXiv_temp](https://fairmat-experimental.github.io/nexus-fairmat-proposal/1c3806dba40111f36a16d0205cc39a5b7d52ca2e/classes/contributed_definitions/NXiv_temp.html#nxiv-temp).
- We use the Nexus ELN features of Nomad to generate a Nexus file.
diff --git a/gui/src/components/About.js b/gui/src/components/About.js
index a5182ae90a..d764a0f940 100644
--- a/gui/src/components/About.js
+++ b/gui/src/components/About.js
@@ -109,12 +109,12 @@ export const DistributionInfo = React.memo(({data}) => {
<li>version: {data.version}</li>
{data?.plugin_packages?.length
? <li>{"plugin packages: "}
- {data.plugin_packages.map(pluginPackage => <>
+ {data.plugin_packages.map((pluginPackage, i) => <>
<Link key={pluginPackage.name} href="#" onClick={() => {
setSelected(pluginPackage)
setTitle('Plugin package')
}}>{pluginPackage.name}</Link>
- {", "}
+ {i !== data.plugin_packages.length - 1 ? ", " : null}
</>)}
</li>
: null
@@ -125,12 +125,14 @@ export const DistributionInfo = React.memo(({data}) => {
const entryPoints = categories[category]
return <li key={category}>
{`${pluralize(category, 2)}: `}
- {entryPoints.map(entryPoint => <>
+ {entryPoints
+ .sort((a, b) => (a.id > b.id) ? 1 : ((b.id > a.id) ? -1 : 0))
+ .map((entryPoint, i) => <>
<Link key={entryPoint.id} href="#" onClick={() => {
setSelected(entryPoint)
setTitle('Plugin entry point')
}}>{entryPoint.id}</Link>
- {", "}
+ {i !== entryPoints.length - 1 ? ", " : null}
</>)}
</li>
})}
diff --git a/gui/src/components/uploads/ExampleUploadButton.js b/gui/src/components/uploads/ExampleUploadButton.js
index 6f58a809ae..f62900bb9c 100644
--- a/gui/src/components/uploads/ExampleUploadButton.js
+++ b/gui/src/components/uploads/ExampleUploadButton.js
@@ -94,7 +94,7 @@ export default function ExampleUploadButton(props) {
const handleClose = () => setOpen(false)
const handleSelect = (exampleUpload) => {
- api.post(`/uploads?local_path=${exampleUpload.local_path || exampleUpload.path}&upload_name=${exampleUpload.title}`)
+ api.post(`/uploads?local_path=${exampleUpload.local_path}&upload_name=${exampleUpload.title}`)
.then((data) => {
history.push(`/user/uploads/upload/id/${data.upload_id}`)
})
diff --git a/gui/src/config.js b/gui/src/config.js
index bc517a67f3..2aec0f25c5 100644
--- a/gui/src/config.js
+++ b/gui/src/config.js
@@ -122,7 +122,7 @@ export const searchQuantities = window.nomadArtifacts.searchQuantities
export const metainfo = window.nomadArtifacts.metainfo
export const parserMetadata = window.nomadArtifacts.parserMetadata
export const toolkitMetadata = window.nomadArtifacts.toolkitMetadata
-export const exampleUploads = window.nomadArtifacts.exampleUploads || {}
+export const exampleUploads = {}
Object.values(entry_points?.options || [])
.filter(entry_point => entry_point.entry_point_type === 'example_upload')
.forEach(entry_point => {
diff --git a/gui/tests/artifacts.js b/gui/tests/artifacts.js
index ff3ceab485..c9eff2fed6 100644
--- a/gui/tests/artifacts.js
+++ b/gui/tests/artifacts.js
@@ -65183,79 +65183,6 @@ window.nomadArtifacts = {
"tableOfFiles": "|Input Filename| Description|\n|--- | --- |\n|`vasprun.xml` | **Mainfile** in plain-text (structured) XML format |\n|`OUTCAR` | plain-text (semi-structured) file, VAPS's detailed output. Read by NOMAD only as fallback to parse `outcar` data |\n"
}
},
- "exampleUploads": {
- "Basic examples": {
- "theory": {
- "path": "examples/data/uploads/theory.zip",
- "title": "Electronic structure code input and output files",
- "description": "This upload demonstrate the basic use of NOMAD's *parsers*. For many *electronic\nstructure codes* (VASP, etc.), NOMAD provides parsers. You simply upload\nthe *input and output files* of your simulations and NOMAD parsers are extracting\nall necessary metadata to produce a **FAIR** dataset.\n"
- },
- "eln": {
- "path": "examples/data/uploads/eln.zip",
- "title": "Electronic Lab Notebook",
- "description": "This example contains a custom NOMAD *schema* to create an **Electronic\nLab Notebook (ELN)** and a few example *data* entries that use this schema.\nThe schema demonstrates the basic concepts behind a NOMAD ELN and can be a good\n**starting point** to create you own schemas that model **FAIR data** acquired in your lab.\n"
- },
- "tables": {
- "path": "examples/data/uploads/tabular.zip",
- "title": "Tabular Data",
- "description": "This upload demonstrates the used of tabular data. In this example we use an *xlsx*\nfile in combination with a custom schema. The schema describes what the columns\nin the excel file mean and NOMAD can parse everything accordingly to\nproduce a **FAIR** dataset.\n"
- }
- },
- "Tutorials": {
- "rdm_tutorial": {
- "path": "examples/data/uploads/rdm_tutorial.zip",
- "title": "Tailored Research Data Management (RDM) with NOMAD",
- "description": "This notebook will teach you how you can build tailored research data\nmanagement (RDM) solutions using NOMAD. It uses existing thermally\nactivated delayed fluorescent (TADF) molecule data to teach you how we\ncan use the NOMAD to turn research data into datasets that are FAIR:\nFindable, Accessible, Interoperable and Reusable. The end-result can be\ndistributed as a NOMAD plugin: a self-contained Python package that can be\ninstalled on a NOMAD Oasis.\n"
- },
- "cow_tutorial": {
- "path": "examples/data/uploads/cow_tutorial.zip",
- "title": "NOMAD as a Data Management Framework Tutorial",
- "description": "This upload provides a notebook as a tutorial that demonstrates how to use NOMAD\nfor managing custom data and file types. Based on a simple *Countries of the World*\ndataset, it shows how to model the data in a schema, do parsing and normalization,\nprocess data, access existing data with NOMAD's API for analysis, and how to\nadd visualization to your data.\n"
- }
- },
- "FAIRmat examples": {
- "ellips": {
- "path": "examples/data/uploads/ellips.zip",
- "title": "Ellipsometry",
- "description": "This example presents the capabilities of the NOMAD platform to store and standardize ellipsometry data.\nIt shows the generation of a NeXus file according to the [NXellipsometry](https://manual.nexusformat.org/classes/contributed_definitions/NXellipsometry.html#nxellipsometry)\napplication definition and a successive analysis of a SiO2 on Si Psi/Delta measurement.\n"
- },
- "mpes": {
- "path": "examples/data/uploads/mpes.zip",
- "title": "Mpes",
- "description": "This example presents the capabilities of the NOMAD platform to store and standardize multi photoemission spectroscopy (MPES) experimental data. It contains three major examples:\n\n - Taking a pre-binned file, here stored in a h5 file, and converting it into the standardized MPES NeXus format.\n There exists a [NeXus application definition for MPES](https://manual.nexusformat.org/classes/contributed_definitions/NXmpes.html#nxmpes) which details the internal structure of such a file.\n - Binning of raw data (see [here](https://www.nature.com/articles/s41597-020-00769-8) for additional resources) into a h5 file and consecutively generating a NeXus file from it.\n - An analysis example using data in the NeXus format and employing the [pyARPES](https://github.com/chstan/arpes) analysis tool to reproduce the main findings of [this paper](https://arxiv.org/pdf/2107.07158.pdf).\n"
- },
- "xps": {
- "path": "examples/data/uploads/xps.zip",
- "title": "XPS",
- "description": "This example presents the capabilities of the NOMAD platform to store and standardize XPS data.\nIt shows the generation of a NeXus file according to the\n[NXmpes](https://manual.nexusformat.org/classes/contributed_definitions/NXmpes.html#nxmpes)\napplication definition and a successive analysis of an example data set.\n"
- },
- "sts": {
- "path": "examples/data/uploads/sts.zip",
- "title": "STS",
- "description": "This example is for two types of experiments: Scanning Tunneling Microscopy (STM) and Scanning Tunneling Spectroscopy (STS) from Scanning Probe Microscopy.\nIt can transform the data from files generated by a the nanonis software into the NeXus application definition NXsts.\nThe example contains data files from the two specific nanonis software versions generic 5e and generic 4.5.\n"
- },
- "stm": {
- "path": "examples/data/uploads/stm.zip",
- "title": "STM",
- "description": "This example is for two types of experiments: Scanning Tunneling Microscopy (STM) and Scanning Tunneling Spectroscopy (STS) from Scanning Probe Microscopy.\nIt can transform the data from files generated by a the nanonis software into the NeXus application definition NXsts.\nThe example contains data files from the two specific nanonis software versions generic 5e and generic 4.5.\n"
- },
- "apm": {
- "path": "examples/data/uploads/apm.zip",
- "title": "Atom Probe Microscopy",
- "description": "This example presents the capabilities of the NOMAD platform to store and standardize atom probe data.\nIt shows the generation of a NeXus file according to the\n[NXapm](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXapm.html#nxapm)\napplication definition and a successive analysis of an example data set.\nThe example contains a small atom probe dataset from an experiment with a LEAP instrument to get you started\nand keep the size of your NOMAD installation small. Ones started, we recommend to change the respective\ninput file in the NOMAD Oasis ELN to run the example with your own datasets.\n"
- },
- "em": {
- "path": "examples/data/uploads/em.zip",
- "title": "Electron Microscopy",
- "description": "This example presents the capabilities of the NOMAD platform to store and standardize electron microscopy.\nIt shows the generation of a NeXus file according to the\n[NXem](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXem.html#nxem)\napplication definition.\nThe example contains a small set of electron microscopy datasets to get started and keep the size of your\nNOMAD installation small. Ones started, we recommend to change the respective input file in the NOMAD Oasis\nELN to run the example with your own datasets.\n"
- },
- "iv_temp": {
- "path": "examples/data/uploads/iv_temp.zip",
- "title": "Sensor Scan - IV Temperature Curve",
- "description": "This example shows users how to take data from a Python framework and map it out to a Nexus application definition for IV Temperature measurements, [NXiv_temp](https://fairmat-experimental.github.io/nexus-fairmat-proposal/1c3806dba40111f36a16d0205cc39a5b7d52ca2e/classes/contributed_definitions/NXiv_temp.html#nxiv-temp).\nWe use the Nexus ELN features of Nomad to generate a Nexus file.\n"
- }
- }
- },
"northTools": {
"jupyter": {
"short_description": "Basic jupyter run with an empty notebook or on given notebook file.",
diff --git a/gui/tests/env.js b/gui/tests/env.js
index bd45124589..6f98f43002 100644
--- a/gui/tests/env.js
+++ b/gui/tests/env.js
@@ -2661,6 +2661,110 @@ window.nomadEnv = {
"description": "Normalizer for the DOS data.",
"plugin_package": "dosnormalizer"
},
+ "example_uploads/1_basic_examples/1_theory": {
+ "id": "example_uploads/1_basic_examples/1_theory",
+ "entry_point_type": "example_upload",
+ "description": "This upload demonstrate the basic use of NOMAD's *parsers*. For many *electronic\nstructure codes* (VASP, etc.), NOMAD provides parsers. You simply upload\nthe *input and output files* of your simulations and NOMAD parsers are extracting\nall necessary metadata to produce a **FAIR** dataset.\n",
+ "category": "Basic examples",
+ "title": "Electronic structure code input and output files",
+ "local_path": "examples/data/uploads/theory.zip"
+ },
+ "example_uploads/1_basic_examples/2_eln": {
+ "id": "example_uploads/1_basic_examples/2_eln",
+ "entry_point_type": "example_upload",
+ "description": "This example contains a custom NOMAD *schema* to create an **Electronic\nLab Notebook (ELN)** and a few example *data* entries that use this schema.\nThe schema demonstrates the basic concepts behind a NOMAD ELN and can be a good\n**starting point** to create you own schemas that model **FAIR data** acquired in your lab.\n",
+ "category": "Basic examples",
+ "title": "Electronic Lab Notebook",
+ "local_path": "examples/data/uploads/eln.zip"
+ },
+ "example_uploads/1_basic_examples/3_tables": {
+ "id": "example_uploads/1_basic_examples/3_tables",
+ "entry_point_type": "example_upload",
+ "description": "This upload demonstrates the used of tabular data. In this example we use an *xlsx*\nfile in combination with a custom schema. The schema describes what the columns\nin the excel file mean and NOMAD can parse everything accordingly to\nproduce a **FAIR** dataset.\n",
+ "category": "Basic examples",
+ "title": "Tabular Data",
+ "local_path": "examples/data/uploads/tabular.zip"
+ },
+ "example_uploads/2_tutorials/1_rdm_tutorial": {
+ "id": "example_uploads/2_tutorials/1_rdm_tutorial",
+ "entry_point_type": "example_upload",
+ "description": "This notebook will teach you how you can build tailored research data\nmanagement (RDM) solutions using NOMAD. It uses existing thermally\nactivated delayed fluorescent (TADF) molecule data to teach you how we\ncan use the NOMAD to turn research data into datasets that are FAIR:\nFindable, Accessible, Interoperable and Reusable. The end-result can be\ndistributed as a NOMAD plugin: a self-contained Python package that can be\ninstalled on a NOMAD Oasis.\n",
+ "category": "Tutorials",
+ "title": "Tailored Research Data Management (RDM) with NOMAD",
+ "local_path": "examples/data/uploads/rdm_tutorial.zip"
+ },
+ "example_uploads/2_tutorials/2_cow_tutorial": {
+ "id": "example_uploads/2_tutorials/2_cow_tutorial",
+ "entry_point_type": "example_upload",
+ "description": "This upload provides a notebook as a tutorial that demonstrates how to use NOMAD\nfor managing custom data and file types. Based on a simple *Countries of the World*\ndataset, it shows how to model the data in a schema, do parsing and normalization,\nprocess data, access existing data with NOMAD's API for analysis, and how to\nadd visualization to your data.\n",
+ "category": "Tutorials",
+ "title": "NOMAD as a Data Management Framework Tutorial",
+ "local_path": "examples/data/uploads/cow_tutorial.zip"
+ },
+ "example_uploads/3_fairmat_examples/1_ellips": {
+ "id": "example_uploads/3_fairmat_examples/1_ellips",
+ "entry_point_type": "example_upload",
+ "description": "This example presents the capabilities of the NOMAD platform to store and standardize ellipsometry data.\nIt shows the generation of a NeXus file according to the [NXellipsometry](https://manual.nexusformat.org/classes/contributed_definitions/NXellipsometry.html#nxellipsometry)\napplication definition and a successive analysis of a SiO2 on Si Psi/Delta measurement.\n",
+ "category": "FAIRmat examples",
+ "title": "Ellipsometry",
+ "local_path": "examples/data/uploads/ellips.zip"
+ },
+ "example_uploads/3_fairmat_examples/2_mpes": {
+ "id": "example_uploads/3_fairmat_examples/2_mpes",
+ "entry_point_type": "example_upload",
+ "description": "This example presents the capabilities of the NOMAD platform to store and standardize multi photoemission spectroscopy (MPES) experimental data. It contains three major examples:\n\n - Taking a pre-binned file, here stored in a h5 file, and converting it into the standardized MPES NeXus format.\n There exists a [NeXus application definition for MPES](https://manual.nexusformat.org/classes/contributed_definitions/NXmpes.html#nxmpes) which details the internal structure of such a file.\n - Binning of raw data (see [here](https://www.nature.com/articles/s41597-020-00769-8) for additional resources) into a h5 file and consecutively generating a NeXus file from it.\n - An analysis example using data in the NeXus format and employing the [pyARPES](https://github.com/chstan/arpes) analysis tool to reproduce the main findings of [this paper](https://arxiv.org/pdf/2107.07158.pdf).\n",
+ "category": "FAIRmat examples",
+ "title": "Mpes",
+ "local_path": "examples/data/uploads/mpes.zip"
+ },
+ "example_uploads/3_fairmat_examples/3_xps": {
+ "id": "example_uploads/3_fairmat_examples/3_xps",
+ "entry_point_type": "example_upload",
+ "description": "This example presents the capabilities of the NOMAD platform to store and standardize XPS data.\nIt shows the generation of a NeXus file according to the\n[NXmpes](https://manual.nexusformat.org/classes/contributed_definitions/NXmpes.html#nxmpes)\napplication definition and a successive analysis of an example data set.\n",
+ "category": "FAIRmat examples",
+ "title": "XPS",
+ "local_path": "examples/data/uploads/xps.zip"
+ },
+ "example_uploads/3_fairmat_examples/4_sts": {
+ "id": "example_uploads/3_fairmat_examples/4_sts",
+ "entry_point_type": "example_upload",
+ "description": "This example is for two types of experiments: Scanning Tunneling Microscopy (STM) and Scanning Tunneling Spectroscopy (STS) from Scanning Probe Microscopy.\nIt can transform the data from files generated by a the nanonis software into the NeXus application definition NXsts.\nThe example contains data files from the two specific nanonis software versions generic 5e and generic 4.5.\n",
+ "category": "FAIRmat examples",
+ "title": "STS",
+ "local_path": "examples/data/uploads/sts.zip"
+ },
+ "example_uploads/3_fairmat_examples/5_stm": {
+ "id": "example_uploads/3_fairmat_examples/5_stm",
+ "entry_point_type": "example_upload",
+ "description": "This example is for two types of experiments: Scanning Tunneling Microscopy (STM) and Scanning Tunneling Spectroscopy (STS) from Scanning Probe Microscopy.\nIt can transform the data from files generated by a the nanonis software into the NeXus application definition NXsts.\nThe example contains data files from the two specific nanonis software versions generic 5e and generic 4.5.\n",
+ "category": "FAIRmat examples",
+ "title": "STM",
+ "local_path": "examples/data/uploads/stm.zip"
+ },
+ "example_uploads/3_fairmat_examples/6_apm": {
+ "id": "example_uploads/3_fairmat_examples/6_apm",
+ "entry_point_type": "example_upload",
+ "description": "This example presents the capabilities of the NOMAD platform to store and standardize atom probe data.\nIt shows the generation of a NeXus file according to the\n[NXapm](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXapm.html#nxapm)\napplication definition and a successive analysis of an example data set.\nThe example contains a small atom probe dataset from an experiment with a LEAP instrument to get you started\nand keep the size of your NOMAD installation small. Ones started, we recommend to change the respective\ninput file in the NOMAD Oasis ELN to run the example with your own datasets.\n",
+ "category": "FAIRmat examples",
+ "title": "Atom Probe Microscopy",
+ "local_path": "examples/data/uploads/apm.zip"
+ },
+ "example_uploads/3_fairmat_examples/7_em": {
+ "id": "example_uploads/3_fairmat_examples/7_em",
+ "entry_point_type": "example_upload",
+ "description": "This example presents the capabilities of the NOMAD platform to store and standardize electron microscopy.\nIt shows the generation of a NeXus file according to the\n[NXem](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXem.html#nxem)\napplication definition.\nThe example contains a small set of electron microscopy datasets to get started and keep the size of your\nNOMAD installation small. Ones started, we recommend to change the respective input file in the NOMAD Oasis\nELN to run the example with your own datasets.\n",
+ "category": "FAIRmat examples",
+ "title": "Electron Microscopy",
+ "local_path": "examples/data/uploads/em.zip"
+ },
+ "example_uploads/3_fairmat_examples/8_iv_temp": {
+ "id": "example_uploads/3_fairmat_examples/8_iv_temp",
+ "entry_point_type": "example_upload",
+ "description": "This example shows users how to take data from a Python framework and map it out to a Nexus application definition for IV Temperature measurements, [NXiv_temp](https://fairmat-experimental.github.io/nexus-fairmat-proposal/1c3806dba40111f36a16d0205cc39a5b7d52ca2e/classes/contributed_definitions/NXiv_temp.html#nxiv-temp).\nWe use the Nexus ELN features of Nomad to generate a Nexus file.\n",
+ "category": "FAIRmat examples",
+ "title": "Sensor Scan - IV Temperature Curve",
+ "local_path": "examples/data/uploads/iv_temp.zip"
+ },
"nomad_aitoolkit.apps:aitoolkit": {
"id": "nomad_aitoolkit.apps:aitoolkit",
"entry_point_type": "app",
diff --git a/nomad/cli/dev.py b/nomad/cli/dev.py
index 61cab37299..bbc4dc8fa3 100644
--- a/nomad/cli/dev.py
+++ b/nomad/cli/dev.py
@@ -87,7 +87,6 @@ def get_gui_artifacts_js() -> str:
'searchQuantities': _generate_search_quantities(),
'metainfo': _generate_metainfo(all_metainfo_packages),
'parserMetadata': code_metadata,
- 'exampleUploads': _generate_example_upload_metadata(),
'northTools': {k: v.dict() for k, v in config.north.tools.filtered_items()},
'unitList': unit_list_json,
'unitPrefixes': prefixes_json,
@@ -272,32 +271,6 @@ def gui_config():
print(get_gui_config())
-def _generate_example_upload_metadata():
- import yaml
-
- example_uploads_path = 'examples/data/uploads/example_uploads.yml'
-
- if not os.path.exists(example_uploads_path):
- example_uploads_path = os.path.join(
- os.path.dirname(__file__),
- '../../',
- 'examples/data/uploads/example_uploads.yml',
- )
-
- if not os.path.exists(example_uploads_path):
- raise FileNotFoundError('Cannot find example_uploads.yml file')
-
- with open(example_uploads_path, 'r') as infile:
- return yaml.load(infile, Loader=yaml.SafeLoader)
-
-
-@dev.command(
- help='Generates a JSON file from example-uploads metadata in the YAML file.'
-)
-def example_upload_metadata():
- print(json.dumps(_generate_example_upload_metadata(), indent=2))
-
-
@dev.command(
help=(
'Updates parser`s README files by combining a general template with '
diff --git a/nomad/config/defaults.yaml b/nomad/config/defaults.yaml
index 848b16574b..7403e0a03c 100644
--- a/nomad/config/defaults.yaml
+++ b/nomad/config/defaults.yaml
@@ -1558,6 +1558,143 @@ plugins:
exclude:
- nomad_porous_materials.normalizers:porositynormalizer
options:
+ example_uploads/1_basic_examples/1_theory:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/theory.zip
+ title: Electronic structure code input and output files
+ category: Basic examples
+ description: |
+ This upload demonstrate the basic use of NOMAD's *parsers*. For many *electronic
+ structure codes* (VASP, etc.), NOMAD provides parsers. You simply upload
+ the *input and output files* of your simulations and NOMAD parsers are extracting
+ all necessary metadata to produce a **FAIR** dataset.
+ example_uploads/1_basic_examples/2_eln:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/eln.zip
+ title: Electronic Lab Notebook
+ category: Basic examples
+ description: |
+ This example contains a custom NOMAD *schema* to create an **Electronic
+ Lab Notebook (ELN)** and a few example *data* entries that use this schema.
+ The schema demonstrates the basic concepts behind a NOMAD ELN and can be a good
+ **starting point** to create you own schemas that model **FAIR data** acquired in your lab.
+ example_uploads/1_basic_examples/3_tables:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/tabular.zip
+ title: Tabular Data
+ category: Basic examples
+ description: |
+ This upload demonstrates the used of tabular data. In this example we use an *xlsx*
+ file in combination with a custom schema. The schema describes what the columns
+ in the excel file mean and NOMAD can parse everything accordingly to
+ produce a **FAIR** dataset.
+ example_uploads/2_tutorials/1_rdm_tutorial:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/rdm_tutorial.zip
+ title: Tailored Research Data Management (RDM) with NOMAD
+ category: Tutorials
+ description: |
+ This notebook will teach you how you can build tailored research data
+ management (RDM) solutions using NOMAD. It uses existing thermally
+ activated delayed fluorescent (TADF) molecule data to teach you how we
+ can use the NOMAD to turn research data into datasets that are FAIR:
+ Findable, Accessible, Interoperable and Reusable. The end-result can be
+ distributed as a NOMAD plugin: a self-contained Python package that can be
+ installed on a NOMAD Oasis.
+ example_uploads/2_tutorials/2_cow_tutorial:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/cow_tutorial.zip
+ title: NOMAD as a Data Management Framework Tutorial
+ category: Tutorials
+ description: |
+ This upload provides a notebook as a tutorial that demonstrates how to use NOMAD
+ for managing custom data and file types. Based on a simple *Countries of the World*
+ dataset, it shows how to model the data in a schema, do parsing and normalization,
+ process data, access existing data with NOMAD's API for analysis, and how to
+ add visualization to your data.
+ example_uploads/3_fairmat_examples/1_ellips:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/ellips.zip
+ title: Ellipsometry
+ category: FAIRmat examples
+ description: |
+ This example presents the capabilities of the NOMAD platform to store and standardize ellipsometry data.
+ It shows the generation of a NeXus file according to the [NXellipsometry](https://manual.nexusformat.org/classes/contributed_definitions/NXellipsometry.html#nxellipsometry)
+ application definition and a successive analysis of a SiO2 on Si Psi/Delta measurement.
+ example_uploads/3_fairmat_examples/2_mpes:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/mpes.zip
+ title: Mpes
+ category: FAIRmat examples
+ description: |
+ This example presents the capabilities of the NOMAD platform to store and standardize multi photoemission spectroscopy (MPES) experimental data. It contains three major examples:
+
+ - Taking a pre-binned file, here stored in a h5 file, and converting it into the standardized MPES NeXus format.
+ There exists a [NeXus application definition for MPES](https://manual.nexusformat.org/classes/contributed_definitions/NXmpes.html#nxmpes) which details the internal structure of such a file.
+ - Binning of raw data (see [here](https://www.nature.com/articles/s41597-020-00769-8) for additional resources) into a h5 file and consecutively generating a NeXus file from it.
+ - An analysis example using data in the NeXus format and employing the [pyARPES](https://github.com/chstan/arpes) analysis tool to reproduce the main findings of [this paper](https://arxiv.org/pdf/2107.07158.pdf).
+ example_uploads/3_fairmat_examples/3_xps:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/xps.zip
+ title: XPS
+ category: FAIRmat examples
+ description: |
+ This example presents the capabilities of the NOMAD platform to store and standardize XPS data.
+ It shows the generation of a NeXus file according to the
+ [NXmpes](https://manual.nexusformat.org/classes/contributed_definitions/NXmpes.html#nxmpes)
+ application definition and a successive analysis of an example data set.
+ example_uploads/3_fairmat_examples/4_sts:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/sts.zip
+ title: STS
+ category: FAIRmat examples
+ description: |
+ This example is for two types of experiments: Scanning Tunneling Microscopy (STM) and Scanning Tunneling Spectroscopy (STS) from Scanning Probe Microscopy.
+ It can transform the data from files generated by a the nanonis software into the NeXus application definition NXsts.
+ The example contains data files from the two specific nanonis software versions generic 5e and generic 4.5.
+ example_uploads/3_fairmat_examples/5_stm:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/stm.zip
+ title: STM
+ category: FAIRmat examples
+ description: |
+ This example is for two types of experiments: Scanning Tunneling Microscopy (STM) and Scanning Tunneling Spectroscopy (STS) from Scanning Probe Microscopy.
+ It can transform the data from files generated by a the nanonis software into the NeXus application definition NXsts.
+ The example contains data files from the two specific nanonis software versions generic 5e and generic 4.5.
+ example_uploads/3_fairmat_examples/6_apm:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/apm.zip
+ title: Atom Probe Microscopy
+ category: FAIRmat examples
+ description: |
+ This example presents the capabilities of the NOMAD platform to store and standardize atom probe data.
+ It shows the generation of a NeXus file according to the
+ [NXapm](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXapm.html#nxapm)
+ application definition and a successive analysis of an example data set.
+ The example contains a small atom probe dataset from an experiment with a LEAP instrument to get you started
+ and keep the size of your NOMAD installation small. Ones started, we recommend to change the respective
+ input file in the NOMAD Oasis ELN to run the example with your own datasets.
+ example_uploads/3_fairmat_examples/7_em:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/em.zip
+ title: Electron Microscopy
+ category: FAIRmat examples
+ description: |
+ This example presents the capabilities of the NOMAD platform to store and standardize electron microscopy.
+ It shows the generation of a NeXus file according to the
+ [NXem](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXem.html#nxem)
+ application definition.
+ The example contains a small set of electron microscopy datasets to get started and keep the size of your
+ NOMAD installation small. Ones started, we recommend to change the respective input file in the NOMAD Oasis
+ ELN to run the example with your own datasets.
+ example_uploads/3_fairmat_examples/8_iv_temp:
+ entry_point_type: example_upload
+ local_path: examples/data/uploads/iv_temp.zip
+ title: Sensor Scan - IV Temperature Curve
+ category: FAIRmat examples
+ description: |
+ This example shows users how to take data from a Python framework and map it out to a Nexus application definition for IV Temperature measurements, [NXiv_temp](https://fairmat-experimental.github.io/nexus-fairmat-proposal/1c3806dba40111f36a16d0205cc39a5b7d52ca2e/classes/contributed_definitions/NXiv_temp.html#nxiv-temp).
+ We use the Nexus ELN features of Nomad to generate a Nexus file.
parsers/abacus:
plugin_type: parser
mainfile_contents_re: \s*\n\s*WELCOME TO ABACUS
diff --git a/nomad/config/models/config.py b/nomad/config/models/config.py
index c8cd39982a..40c4147c2f 100644
--- a/nomad/config/models/config.py
+++ b/nomad/config/models/config.py
@@ -1142,15 +1142,20 @@ class Config(ConfigBaseModel):
for key, plugin in _plugins['entry_points']['options'].items():
if key not in plugin_entry_point_ids:
- plugin_config = load_plugin_yaml(key, plugin)
- plugin_config['id'] = key
- plugin_class = {
- 'parser': Parser,
- 'normalizer': Normalizer,
- 'schema': Schema,
- }.get(plugin_config['plugin_type'])
- _plugins['entry_points']['options'][key] = plugin_class.parse_obj(
- plugin_config
- )
+ # Update information for old style plugins
+ if plugin.get('plugin_type'):
+ plugin_config = load_plugin_yaml(key, plugin)
+ plugin_config['id'] = key
+ plugin_class = {
+ 'parser': Parser,
+ 'normalizer': Normalizer,
+ 'schema': Schema,
+ }.get(plugin_config['plugin_type'])
+ _plugins['entry_points']['options'][key] = (
+ plugin_class.parse_obj(plugin_config)
+ )
+ # Handle new style plugins that are declared directly in nomad.yaml
+ elif not plugin.get('id'):
+ plugin['id'] = key
self.plugins = Plugins.parse_obj(_plugins)
diff --git a/nomad/config/models/plugins.py b/nomad/config/models/plugins.py
index fb8e94e7dc..c9800437d1 100644
--- a/nomad/config/models/plugins.py
+++ b/nomad/config/models/plugins.py
@@ -215,7 +215,8 @@ class ExampleUploadEntryPoint(EntryPoint):
local_path: Optional[str] = Field(
description="""
The final path to use when creating the upload. This field will be
- automatically generated by the 'load' function.
+ automatically generated by the 'load' function and is typically not set
+ manually.
"""
)
@@ -224,10 +225,11 @@ class ExampleUploadEntryPoint(EntryPoint):
"""Checks that only either path or url is given."""
path = values.get('path')
url = values.get('url')
+ local_path = values.get('local_path')
if path and url:
raise ValueError('Provide only "path" or "url", not both.')
- if not path and not url:
- raise ValueError('Provide either "path" or "url".')
+ if not path and not url and not local_path:
+ raise ValueError('Provide one of "path", "url" or "local_path".')
return values
@@ -245,6 +247,10 @@ class ExampleUploadEntryPoint(EntryPoint):
startup.
"""
path = self.path
+ # If local path is already set, use it
+ if self.local_path:
+ return
+ # Create local path from given path or url
if not path and self.url:
final_folder = os.path.join(
get_package_path(self.plugin_package), 'example_uploads'
diff --git a/tests/config/models/test_plugins.py b/tests/config/models/test_plugins.py
index c96f2e1c8e..9c9018245e 100644
--- a/tests/config/models/test_plugins.py
+++ b/tests/config/models/test_plugins.py
@@ -81,8 +81,8 @@ def test_example_upload_entry_point_valid(config, expected_local_path, monkeypat
'description': 'test',
'category': 'test',
},
- 'Provide either "path" or "url".',
- id='no path or url given',
+ 'Provide one of "path", "url" or "local_path".',
+ id='no path, url or local_path given',
),
pytest.param(
{
--
GitLab