Commit 5e8b9dbe authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.9.4' into 'master'

Merge for release v0.9.4

See merge request !209
parents 972071db 674a3390
Pipeline #86204 passed with stages
in 53 seconds
Subproject commit 8ca8b53425e2808a80b69c8397a9cd977bc08f7e
Subproject commit 0ca60338acd93d10f70fc66f5a64bec422613f9d
{
"name": "nomad-fair-gui",
"version": "0.9.3",
"version": "0.9.4",
"commit": "e98694e",
"private": true,
"dependencies": {
......
......@@ -9,7 +9,7 @@ window.nomadEnv = {
'matomoUrl': 'https://nomad-lab.eu/fairdi/stat',
'matomoSiteId': '2',
'version': {
'label': '0.9.3',
'label': '0.9.4',
'isBeta': false,
'isTest': true,
'usesBetaData': true,
......
......@@ -134,7 +134,9 @@ export function metainfoAdaptorFactory(obj) {
class MetainfoAdaptor extends Adaptor {
itemAdaptor(key) {
if (key.startsWith('_category:')) {
if (key === '_reference') {
return metainfoAdaptorFactory(resolveRef(this.e.type.type_data))
} else if (key.startsWith('_category:')) {
const categoryName = key.split(':')[1]
return metainfoAdaptorFactory(this.e.categories.map(ref => resolveRef(ref)).find(categoryDef => categoryDef.name === categoryName))
} else if (key === '_metainfo') {
......@@ -216,8 +218,13 @@ export class PackagePrefixAdaptor extends MetainfoAdaptor {
function Metainfo(props) {
return <Content>
<Compartment title="root sections">
{rootSections.map(def => (
<Compartment title="archive root section">
<Item itemKey="EntryArchive">
<Typography>EntryArchive</Typography>
</Item>
</Compartment>
<Compartment title="other root sections">
{rootSections.filter(def => def.name !== 'EntryArchive').map(def => (
<Item key={def.name} itemKey={def.name}>
<Typography>
{def.name}
......@@ -263,6 +270,7 @@ class CategoryDefAdaptor extends MetainfoAdaptor {
render() {
return <Content>
<Definition def={this.e} />
<DefinitionDetails def={this.e} />
</Content>
}
}
......@@ -324,6 +332,7 @@ function SectionDef({def}) {
})
}
</Compartment>
<DefinitionDetails def={def} />
</Content>
}
SectionDef.propTypes = ({
......@@ -333,6 +342,28 @@ SectionDef.propTypes = ({
function QuantityDef({def}) {
return <Content>
<Definition def={def} kindLabel="quantity definition"/>
<Compartment title="properties">
{def.type.type_kind !== 'reference'
? <Typography>
<b>type</b>:&nbsp;
{def.type.type_data}&nbsp;
{def.type.type_kind !== 'data' && `(${def.type.type_kind})`}
</Typography>
: <Item itemKey="_reference">
<Typography>
<b>referenced section</b>
</Typography>
</Item>}
<Typography>
<b>shape</b>:&nbsp;
[{def.shape.join(', ')}]
</Typography>
{def.unit &&
<Typography><b>unit</b>:&nbsp;{def.unit}</Typography>}
{def.aliases && def.aliases !== [] && <Typography><b>aliases</b>:&nbsp;{def.aliases.map(a => `"${a}"`).join(', ')}</Typography>}
{def.derived && <Typography><b>derived</b></Typography>}
</Compartment>
<DefinitionDetails def={def} />
</Content>
}
QuantityDef.propTypes = ({
......@@ -340,6 +371,22 @@ QuantityDef.propTypes = ({
})
function Definition({def, ...props}) {
return <React.Fragment>
<Title def={def} isDefinition {...props} />
{def.description && !def.extends_base_section &&
<Compartment title="description">
<Box marginTop={1} marginBottom={1}>
<Markdown>{def.description}</Markdown>
</Box>
</Compartment>
}
</React.Fragment>
}
Definition.propTypes = {
def: PropTypes.object.isRequired
}
function DefinitionDetails({def, ...props}) {
const {api} = useContext(apiContext)
const lane = useContext(laneContext)
const isLast = !lane.next
......@@ -356,14 +403,14 @@ function Definition({def, ...props}) {
}, [api, def.name, setUsage])
return <React.Fragment>
<Title def={def} isDefinition {...props} />
{isLast && def.description && !def.extends_base_section &&
<Compartment title="description">
<Box marginTop={1} marginBottom={1}>
<Markdown>{def.description}</Markdown>
</Box>
</Compartment>
}
{def.categories && def.categories.length > 0 && <Compartment title="Categories">
{def.categories.map(categoryRef => {
const categoryDef = resolveRef(categoryRef)
return <Item key={categoryRef} itemKey={'_category:' + categoryDef.name}>
<Typography>{categoryDef.name}</Typography>
</Item>
})}
</Compartment>}
{isLast && !def.extends_base_section && def.name !== 'EntryArchive' &&
<Compartment title="graph">
<VicinityGraph def={def} />
......@@ -388,17 +435,9 @@ function Definition({def, ...props}) {
)}
</Compartment>
}
{def.categories && def.categories.length > 0 && <Compartment title="Categories">
{def.categories.map(categoryRef => {
const categoryDef = resolveRef(categoryRef)
return <Item key={categoryRef} itemKey={'_category:' + categoryDef.name}>
<Typography>{categoryDef.name}</Typography>
</Item>
})}
</Compartment>}
</React.Fragment>
}
Definition.propTypes = {
DefinitionDetails.propTypes = {
def: PropTypes.object.isRequired
}
......
......@@ -271,7 +271,7 @@ datacite = NomadConfig(
)
meta = NomadConfig(
version='0.9.3',
version='0.9.4',
commit=gitinfo.commit,
release='devel',
default_domain='dft',
......
......@@ -5102,6 +5102,7 @@ class section_system(MSection):
atom_labels = Quantity(
type=str,
shape=['number_of_atoms'],
aliases=['elements'],
description='''
Labels of the atoms. These strings identify the atom kind and conventionally start
with the symbol of the atomic species, possibly followed by the atomic number. The
......
......@@ -48,18 +48,18 @@ Starting example
pbc = Quantity(type=bool, shape=[3])
class Run(MSection):
systems = SubSection(sub_section=System, repeats=True)
section_system = SubSection(sub_section=System, repeats=True)
We define simple metainfo schema with two `sections` called ``System`` and ``Run``. Sections
allow to organize related data into, well, `sections`. Each section can have two types of
allow to organize related data into, well, `sections`. Each section can have two types of
properties: `quantities` and `sub-sections`. Sections and their properties are defined with
Python classes and their attributes.
Each `quantity` defines a piece of data. Basic quantity attributes are its `type`, `shape`,
`unit`, and `description`.
`Sub-sections` allow to place section into each other and there allow to form containment
`Sub-sections` allow to place section into each other and therefore allow to form containment
hierarchies or sections and the respective data in them. Basic sub-section attributes are
`sub_section`(i.e. a reference to the section definition of the sub-section) and `repeats`
(determines if a sub-section can be contained once or multiple times).
......@@ -107,8 +107,45 @@ This will serialize the data into JSON:
]
}
Definitions
-----------
Definitions and Instances
-------------------------
As you already saw in the example, we first need to define what data can look like (schema),
before we can actually program with data. Because schema and data are often discussed in the
same context, it is paramount to clearly distingish between both. For example, if we
just say "system", it is unclear what we refer to. We could mean the idea of a system, i.e. all
possible systems, a data structure that comprises a lattice, atoms with their elements and
positions in the lattice. Or we mean a specific system of a specific calculation, with
a concrete set of atoms, real numbers for lattice vectors and atoms positions as concrete data.
The NOMAD Metainfo is just a collection of definition that describe what materials science
data could be (a schema). The NOMAD Archive is all the data that we extract from all data
provided to NOMAD. The data in the NOMAD Archive follows the definitions of the NOMAD metainfo.
Similarely, we need to distingish between the NOMAD Metainfo as a collection of definitions
and the Metainfo system that defines how to define a section or a quantity. In this sense,
we have a three layout model, were the Archive (data) is an instance of the Metainfo (schema)
and the Metainfo is an instance of the Metainfo system (schema of the schema).
This documentation describes the Metainfo by explaining the means of how to write down definitions
in Python. Conceptually we map the Metainfo system to Python language constructs, e.g.
a section definition is a Python class, a quantity a Python property, etc. If you are
familiar with databases, this is similar to what an object relational mapping (ORM) would do.
Common attributes of Metainfo Definitions
-----------------------------------------
In the example, you already saw the basic Python interface to the Metainfo. *Sections* are
represented in Python as objects. To define a section, you write a Python classes that inherits
from ``MSection``. To define sub-sections and quantities you use Python properties. The
definitions themselves are also objects derived from classes. For sub-sections and
quantities, you directly instantiate :class:`SubSection` and :class`Quantity`. For sections
there is a generated object derived from :class:`Section` that is available via
`m_def` from each `section class` and `section instance`.
These Python classes that are used to represent metainfo definitions form an inheritance
hierarchy to share common properties
.. autoclass:: Definition
......@@ -116,66 +153,45 @@ Definitions
Quantities
----------
.. autoclass:: Quantity
.. _metainfo-sections:
Quantity definitions are the main building block of meta-info schemas. Each quantity
represents a single piece of data.
Sections
--------
.. autoclass:: Quantity
With sections it is paramount to always be clear what is talked about. The lose
term `section` can reference one of the following three:
* `section definition`
Which is a Python object that represents the definition of
a section, its sub-sections and quantities. `Section definitions` should not be not
written directly. `Section definitions` are objects of :class:`Section`.
* `secton class`
Which is a Python class and :class:`MSection` decendant that is
used to express a `section defintion` in Python. Each `section class` is tightly
associated with its `section definition`. The `section definition` can be access
with the class attribute ``m_def``. The `section definition` is automatically created
from the `section class` upon defining the class through metaclass vodoo.
* `section instance`
The instance (object) of a `section class`, it `follows` the
definition associated with the instantiated `section class`. The followed
section definition can be accessed with the object attribute ``m_def``.
A `section class` looks like this:
.. code-block:: python
Sections and Sub-Sections
-------------------------
class SectionName(BaseSection):
\'\'\' Section description \'\'\'
m_def = Section(**section_attributes)
.. _metainfo-sections:
quantity_name = Quantity(**quantity_attributes)
sub_section_name = SubSection(**sub_section_attributes)
The various Python elements of this class are mapped to a respective *section definition*.
The ``SectionName`` becomes the *name*. The
``BaseSection`` is either :class:`MSection` or another *section class*. The
``section_attributes`` become additional attributes of the `section definition`. The
various ``Quantity`` and ``SubSection`` become the *quantities* and *sub sections*.
The NOMAD Metainfo allows to create hierarchical (meta-)data structures. A hierarchy
is basically a tree, where *sections* make up the root and inner nodes of the tree,
and *quantities* are the leaves of the tree. In this sense a section can hold further
sub-sections (more branches of the tree) and quantities (leaves). We say a section can
have two types of *properties*: *sub-sections* and *quantities*.
Each *section class* has to directly or indirectly extend :class:`MSection`. This will
provided certain class and object features to all *section classes* and all *section instances*.
Read :ref:metainfo-reflection to learn more.
There is a clear distinction between *section* and *sub-section*. The term *section*
refers to an object that holds data (*properties*) and the term *sub-section* refers to a
relation between *sections*, i.e. between the containing section and the contained
sections of the same type. Furthermore, we have to distinguish between *sections* and
*section definitions*, as well as *sub-sections* and *sub-section definitions*. A *section
definition* defines the possible properties of its instances, and an instantiating *section*
can store quantities and sub-sections according to its definition. A *sub-section*
definition defines a principle containment relationship between two *section definitions*
and a *sub-section* is a set of *sections* (contents) contained in another *section* (container).
*Sub-section definitions* are parts (*properties*) of the definition of containing sections.
.. autoclass:: Section
Sub-Sections
------------
.. autoclass:: SubSection
.. _metainfo-categories:
Categories
----------
.. autoclass:: Quantity
In the old meta-info this was known as `abstract types`.
Categories are defined with Python classes that have :class:`MCategory` as base class.
......@@ -210,6 +226,7 @@ Custom data types
.. autoclass:: MEnum
.. _metainfo-reflection:
Reflection and custom data storage
......@@ -253,7 +270,7 @@ object when a respective quantity is accessed.
.. autoclass:: MProxy
Resources
_________
---------
.. autoclass:: MResource
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
''' An example metainfo package. '''
import numpy as np
......@@ -63,10 +49,9 @@ class System(MSection):
lattice_vectors = Quantity(
type=np.dtype('f'), shape=[3, 3], unit=ureg.m, categories=[SystemHash],
aliases=['unit_cell'],
description='The lattice vectors of the simulated unit cell.')
unit_cell = Quantity(synonym_for='lattice_vectors')
periodic_dimensions = Quantity(
type=bool, shape=[3], default=[False, False, False], categories=[SystemHash],
description='A vector of booleans indicating in which dimensions the unit cell is repeated.')
......
This diff is collapsed.
## Containers for additional services
Some third party services cannot be used with ready made public dockerhub images. For those
we provide our own Dockerfiles that extend public base images. These are largely due to
NOMAD specific configuration that we want to apply to the the base images.
### Keycloak (optional)
A custom version of [jboss/keycloak](https://hub.docker.com/r/jboss/keycloak/)
- added bcrypt password hashing: [https://github.com/leroyguillaume/keycloak-bcrypt](https://github.com/leroyguillaume/keycloak-bcrypt)
- create admin user if not there
- import test realm on first use
- use H2 database
- change config to allow reverse proxy under custom prefix
### CI runner (optional)
This is the immage that this project uses for its gitlab-ci runner. To build an
push it, you have to log into the project's registry (see [gitlab docs](https://docs.gitlab.com/ee/user/packages/container_registry/)) and do
```
cd ci-runner
docker build -t gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner .
docker push gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
```
This image allows to bash, git, docker, docker-compose, k8s, and helm3.
FROM docker/compose
RUN apk update && apk add --no-cache git openssh gettext
ARG VCS_REF
ARG BUILD_DATE
# Metadata
LABEL org.label-schema.vcs-ref=$VCS_REF \
org.label-schema.name="helm-kubectl" \
org.label-schema.url="https://hub.docker.com/r/dtzar/helm-kubectl/" \
org.label-schema.vcs-url="https://github.com/dtzar/helm-kubectl" \
org.label-schema.build-date=$BUILD_DATE
# Note: Latest version of kubectl may be found at:
# https://github.com/kubernetes/kubernetes/releases
ENV KUBE_LATEST_VERSION="v1.18.0"
# Note: Latest version of helm may be found at:
# https://github.com/kubernetes/helm/releases
ENV HELM_VERSION="v3.0.3"
RUN apk add --no-cache ca-certificates bash git openssh curl \
&& wget -q https://storage.googleapis.com/kubernetes-release/release/${KUBE_LATEST_VERSION}/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl \
&& chmod +x /usr/local/bin/kubectl \
&& wget -q https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz -O - | tar -xzO linux-amd64/helm > /usr/local/bin/helm \
&& chmod +x /usr/local/bin/helm
WORKDIR /config
CMD bash
\ No newline at end of file
FROM jboss/keycloak:7.0.0
ARG admin_password=password
ARG prefix='fairdi\/keycloak'
USER jboss
# Change the 'web-context' to allow reverse proxy behind custom path
RUN echo "s/<web-context>auth<\\/web-context>/<web-context>$prefix\\/auth<\\/web-context>/"
RUN sed -i -e "s/<web-context>auth<\\/web-context>/<web-context>$prefix\\/auth<\\/web-context>/" $JBOSS_HOME/standalone/configuration/standalone.xml
RUN sed -i -e "s/<web-context>auth<\\/web-context>/<web-context>$prefix\\/auth<\\/web-context>/" $JBOSS_HOME/standalone/configuration/standalone-ha.xml
RUN sed -i -e "s/name=\"\\/\"/name=\"\\/$prefix\\/\"/" $JBOSS_HOME/standalone/configuration/standalone.xml
RUN sed -i -e "s/name=\"\\/\"/name=\"\\/$prefix\\/\"/" $JBOSS_HOME/standalone/configuration/standalone-ha.xml
RUN sed -i -e "s/\\/auth/\\/$prefix\\/auth/" $JBOSS_HOME/welcome-content/index.html
ENV KEYCLOAK_USER=admin
ENV KEYCLOAK_PASSWORD=$admin_password
ENV KEYCLOAK_IMPORT=/opt/jboss/keycloak/fairdi_nomad_test.json
ENV PROXY_ADDRESS_FORWARDING=true
ENV DB_VENDOR=h2
# Add exported real/client configurations
ADD ./fairdi_nomad_test.json /opt/jboss/keycloak/fairdi_nomad_test.json
ADD ./fairdi_nomad_prod.json /opt/jboss/keycloak/fairdi_nomad_prod.json
# Add a custom theme to mimic the rest of nomad's GUI
ADD ./material_theme /opt/jboss/keycloak/themes/material
# Add custom registration validation
ADD ./registration_form_action/target/form-extension-1.0-SNAPSHOT.jar /opt/jboss/keycloak/standalone/deployments/form-extension-1.0-SNAPSHOT.jar
# Additing a bcrypt password hashing algorithm to reuse old repo users
ADD ./jbcrypt-0.4.jar /tmp/jbcrypt-0.4.jar
RUN /opt/jboss/keycloak/bin/jboss-cli.sh --command="module add --name=org.mindrot.jbcrypt --resources=/tmp/jbcrypt-0.4.jar"
ADD ./keycloak-bcrypt-1.1.0.jar /opt/jboss/keycloak/standalone/deployments/keycloak-bcrypt-1.1.0.jar
## Introduction
This basically uses the "official" 7.0.0 keycloak with some additions
- material theme
- bcrypt support (https://github.com/leroyguillaume/keycloak-bcrypt/)
- a custom form action to check the username (that has to be build before building the image!)
- realm definitions for prod and test realm
- scripts for import and export of all users
## To build the image
```
cd registration_form_action
mvn package
cd ..
docker build -t nomad/keycloak .
```
## Running, Developing, Debugging
To run a keycloak container for development/test use the following. The volume mount
will allow you to edit the theme files while keycloak is running.
```
docker run -p 8002:8080 -v `pwd`/material_theme:/opt/jboss/keycloak/themes/material nomad/keycloak
```
\ No newline at end of file
docker exec -it nomad_keycloak keycloak/bin/standalone.sh \
-Djboss.socket.binding.port-offset=100 \
-Dkeycloak.migration.action=export \
-Dkeycloak.migration.provider=singleFile \
-Dkeycloak.migration.realmName=fairdi_nomad_prod \
-Dkeycloak.migration.usersExportStrategy=REALM_FILE \
-Dkeycloak.migration.file=/export/fairdi_nomad_prod_latest.json
This diff is collapsed.
This diff is collapsed.
docker exec -it nomad_keycloak keycloak/bin/standalone.sh \
-Djboss.socket.binding.port-offset=200 \
-Dkeycloak.migration.action=import \
-Dkeycloak.migration.provider=singleFile \
-Dkeycloak.migration.realmName=fairdi_nomad_prod \
-Dkeycloak.migration.strategy=OVERWRITE_EXISTING \
-Dkeycloak.migration.file=/export/fairdi_nomad_prod_latest.json
<#import "template.ftl" as layout>
<@layout.registrationLayout; section>
<#if section = "title">
${msg("Edit your NOMAD account",(realm.displayName!''))?no_esc}
<#elseif section = "header">
<div class="title">
${msg("Edit your NOMAD account",(realm.displayNameHtml!''))?no_esc}
</div>
<#elseif section = "form">
<form id="kc-register-form" class="register form ${properties.kcFormClass!}" action="${url.accountUrl}" class="form-horizontal" method="post">
<input type="text" readonly value="this is not a login form" style="display: none;">
<input type="password" readonly value="this is not a login form" style="display: none;">
<input type="hidden" id="stateChecker" name="stateChecker" value="${stateChecker}">
<input type="hidden" id="username" name="username" value="${(account.username!'')}">
<div class="mdc-text-field mdc-text-field--outlined mdc-text-field--with-leading-icon ${properties.kcLabelClass!}">
<i class="material-icons mdc-text-field__icon" tabindex="-1" role="button">person</i>
<input required id="firstName" class="mdc-text-field__input ${properties.kcInputClass!}" name="firstName" type="text" autofocus value="${(account.firstName!'')}">
<div class="${properties.kcLabelWrapperClass!}">
<label for="firstName" class="mdc-floating-label ${properties.kcLabelClass!}">${msg("firstName")?no_esc}</label>
</div>
<div class="mdc-notched-outline">
<svg>
<path class="mdc-notched-outline__path"/>
</svg>
</div>
<div class="mdc-notched-outline__idle"></div>
</div>
<div class="mdc-text-field mdc-text-field--outlined mdc-text-field--with-leading-icon ${properties.kcLabelClass!}">
<i class="material-icons mdc-text-field__icon" tabindex="-1" role="button">person</i>
<input required id="lastName" class="mdc-text-field__input ${properties.kcInputClass!}" name="lastName" type="text" value="${(account.lastName!'')}">
<div class="${properties.kcLabelWrapperClass!}">
<label for="lastName" class="mdc-floating-label ${properties.kcLabelClass!}">${msg("lastName")?no_esc}</label>
</div>
<div class="mdc-notched-outline">
<svg>
<path class="mdc-notched-outline__path"/>
</svg>
</div>
<div class="mdc-notched-outline__idle"></div>
</div>
<div class="mdc-text-field mdc-text-field--outlined mdc-text-field--with-leading-icon ${properties.kcLabelClass!}">
<i class="material-icons mdc-text-field__icon" tabindex="-1" role="button">email</i>
<input required id="email" class="mdc-text-field__input ${properties.kcInputClass!}" name="email" type="text" value="${(account.email!'')}">
<div class="${properties.kcLabelWrapperClass!}">
<label for="email" class="mdc-floating-label ${properties.kcLabelClass!}">${msg("email")?no_esc}</label>
</div>
<div class="mdc-notched-outline">
<svg>
<path class="mdc-notched-outline__path"/>
</svg>
</div>
<div class="mdc-notched-outline__idle"></div>
</div>
<div class="mdc-text-field mdc-text-field--outlined mdc-text-field--with-leading-icon ${properties.kcLabelClass!}">
<i class="material-icons mdc-text-field__icon" tabindex="-1" role="button">room</i>
<input required id="user.attributes.affiliation" class="mdc-text-field__input ${properties.kcInputClass!}" name="user.attributes.affiliation" type="text" value="${(account.attributes.affiliation!'')}">
<div class="${properties.kcLabelWrapperClass!}">
<label for="user.attributes.affiliation" class="mdc-floating-label ${properties.kcLabelClass!}">Affiliation</label>
</div>
<div class="mdc-notched-outline">
<svg>
<path class="mdc-notched-outline__path"/>
</svg>
</div>
<div class="mdc-notched-outline__idle"></div>
</div>
<div class="mdc-text-field mdc-text-field--outlined mdc-text-field--with-leading-icon ${properties.kcLabelClass!}">
<i class="material-icons mdc-text-field__icon" tabindex="-1" role="button">account_balance</i>
<input required id="user.attributes.affiliation_address" class="mdc-text-field__input ${properties.kcInputClass!}" name="user.attributes.affiliation_address" type="text" value="${(account.attributes.affiliation_address!'')}">
<div class="${properties.kcLabelWrapperClass!}">
<label for="user.attributes.affiliation_address" class="mdc-floating-label ${properties.kcLabelClass!}">Affiliation address</label>
</div>
<div class="mdc-notched-outline">
<svg>
<path class="mdc-notched-outline__path"/>