Merge branch 'main' into 'master'

moving from github

See merge request !1

.dockerignore

+**/.git
+
+# mongodb instance folder
+db/
+
+# results of the the notebooks
+tutorials/flow_sic_relax
+tutorials/launcher_*/
+

.gitignore

+
+# mongodb instance folder
+db/
+
+# results of the the notebooks
+tutorials/flow_sic_relax
+tutorials/launcher_*/
+
+# VSCode IDE
+.vscode
+
+# https://github.com/github/gitignore/blob/master/Global/macOS.gitignore
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# https://github.com/github/gitignore/blob/master/Python.gitignore
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
\ No newline at end of file

Dockerfile

+# https://github.com/jupyter/docker-stacks
+# https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html
+
+FROM jupyter/scipy-notebook:4d9c9bd9ced0 AS builder
+
+# Abinit compilation
+# ==================
+# # 1. compiler: gfortran
+# # 2. MPI libraries - choice for Open MPI: mpi-default-dev libopenmpi-dev
+# # 3. math libraries - choice for lapack and blas: liblapack-dev libblas-dev
+# # 4. mandatory libraries:libhdf5-dev libnetcdf-dev libnetcdff-dev libpnetcdf-dev libxc-dev libfftw3-dev libxml2-dev
+
+USER root
+
+RUN apt-get update \
+ && apt install -y --no-install-recommends \
+    gfortran \
+    mpi-default-dev libopenmpi-dev \
+    liblapack-dev libblas-dev \
+    libhdf5-dev libnetcdf-dev libnetcdff-dev libpnetcdf-dev libxc-dev \
+    libfftw3-dev libxml2-dev \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /tmp
+
+ARG abinit_version="9.4.0"
+COPY configs/abinit/abinit_config.ac9 /tmp/abinit-${abinit_version}/build/abinit_config.ac9
+
+RUN wget -k "https://www.abinit.org/sites/default/files/packages/abinit-${abinit_version}.tar.gz" \
+ && tar xzf abinit-${abinit_version}.tar.gz \
+ && cd abinit-${abinit_version} \
+ && cd build \
+ && ../configure --with-config-file='abinit_config.ac9'  --prefix=/opt/abinit \
+ && make -j4 \
+ && make install
+
+
+FROM jupyter/scipy-notebook
+LABEL maintainer="Gian-Marco Rignanese <gian-marco.rignanese@uclouvain.be>"
+
+
+# Setup slurm
+# ===========
+
+USER root
+
+RUN apt-get update \
+ && apt install -y --no-install-recommends \
+    slurmd slurm-client slurmctld \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY configs/slurm/slurm.conf /etc/slurm-llnl/slurm.conf
+COPY configs/slurm/start_slurm.sh  /usr/local/bin/before-notebook.d/
+
+# https://github.com/yuwata/slurm-fedora/blob/master/slurm-setuser.in
+RUN mkdir -p /run/munge \
+ && chown -R jovyan /run/munge /etc/munge /var/lib/munge /var/log/munge \
+ && mkdir -p /var/run/slurm-llnl \
+ && chown -R jovyan /var/run/slurm-llnl /var/lib/slurm-llnl /var/log/slurm-llnl
+
+USER $NB_UID
+
+
+# Abinit installation
+# ===================
+# - MPI libraries - choice for Open MPI: mpi-default-bin openmpi-bin libopenmpi3
+# - math libraries - choice for lapack and blas: liblapack3 libblas3
+# - mandatory libraries: libhdf5-103 libnetcdf15 libnetcdff7 libpnetcdf0d libxc5 libfftw3-bin libxml2
+
+USER root
+
+RUN apt-get update \
+ && apt install -y --no-install-recommends \
+    mpi-default-bin openmpi-bin libopenmpi3 \
+    liblapack3 libblas3 \
+    libhdf5-103 libnetcdf15 libnetcdff7 libpnetcdf0d libxc5 \
+    libfftw3-bin libxml2 \
+ && rm -rf /var/lib/apt/lists/*
+
+USER $NB_UID
+
+COPY --chown=$NB_UID:$NB_GID --from=builder /opt/abinit /opt/abinit
+ENV PATH=/opt/abinit/bin:$PATH
+
+
+# Install Python 3 packages
+# =========================
+# Depenecies:
+# - fireworks: flask-paginate gunicorn pymongo
+# - pseudo_dojo: periodic-table-plotter atomicfile
+
+RUN conda install --quiet --yes \
+    'abipy' \
+    'jupyter-server-proxy' \
+    'flask-paginate' 'gunicorn' 'pymongo' \
+    'periodic-table-plotter' 'atomicfile' \
+ && pip install --no-cache-dir jupyter-jsmol \
+ && pip install --no-cache-dir git+https://github.com/modl-uclouvain/fireworks.git \
+ && pip install --no-cache-dir jupyter-fireworks-proxy \
+ && pip install --no-cache-dir git+https://github.com/gpetretto/abiflows.git@develop \
+ && conda clean --all -f -y \
+ && fix-permissions "${CONDA_DIR}" \
+ && fix-permissions "/home/${NB_USER}"
+
+# Install a "lightweight" version of Pseudo-dojo
+COPY --chown=$NB_UID:$NB_GID pseudo_dojo /opt/pseudo_dojo
+WORKDIR /opt/pseudo_dojo
+RUN pip install -e .
+
+# Setup home folder 
+# =================
+
+WORKDIR $HOME
+
+COPY --chown=$NB_UID:$NB_GID configs/abipy .abinit/abipy
+COPY --chown=$NB_UID:$NB_GID configs/fireworks .fireworks
+
+COPY --chown=$NB_UID:$NB_GID tutorials tutorials

README.md

-# workflows-workshop
+# Single-user Jupyter notebook image for workshops
+
+**Please note that this documentation is just a draft. Some of the instructions may be incomplete.**
+
+This notebook folder contains Dockerfile for a single user notebook image which based on Jupyter docker stack.
+In addition to the base notebook, it contains all of the necessary packages for the tools. 
+The docker image is suitable for running/testing locally and for deploying it by the jupyterhub on a kubernetes cluster.
+
+## Short description
+
+- Based on the jupyter docker image
+- Compiled abinit
+- Conda python environments 
+
+## Building/using your own docker image
+
+```bash
+git clone https://github.com/modl-uclouvain/workflows-workshop.git
+cd workflows-workshop
+```
+
+### Building the docker image locally
+
+```bash
+docker build -t workflows-workshop:latest .
+```
+
+### Testing/Developing the notebooks locally
+- Use the following command to run the docker image locally:
+  ```bash
+  docker run --rm \
+             -p 8888:8888 \
+             -v $PWD/tutorials:/home/jovyan/tutorials \
+             --name workflows-workshop \
+             workflows-workshop:latest 
+  ```
+  Note: Although the `--rm` option is useful, you have to use it very carefully. When you stop the notebook server, you can lose all of your modifications which hasn't been stored into the mounted folder.
+
+- To attach a terminal to the running container, you can use the following command:
+  ```bash
+  docker exec -it workflows-workshop start.sh
+  ```
+  
+More info: https://jupyter-docker-stacks.readthedocs.io/en/latest/using/common.html?highlight=root#alternative-commands
+    
+    
+### Useful tricks for Linux
+
+- For mounting a folder, you may need to use an absolute path or other tricks if the PWD environmental variable is not accessible in your shell:
+  ```bash
+  docker run --rm \
+             -p 8888:8888  \
+             -v /path/for/the/tutorials:/home/jovyan/tutorials  \
+             --name workflows-workshop \
+             workflows-workshop:latest
+  ```
+- you may need to change the user id in the container - by adding `--user root` and `-e NB_UID=1001` options to your command - to have access for the mounted folders:
+  ```bash
+  docker run --rm \
+             --user root \
+             -e NB_UID=1001 \
+             -p 8888:8888  \
+             -v $PWD/tutorials:/home/jovyan/tutorials  \
+             --name workflows-workshop \
+             workflows-workshop:latest
+  ```
+- you can have a password-less sudo access in the container for debugging by adding `--user root` and `-e GRANT_SUDO=yes` options to your command:
+  ```bash
+  docker run --rm \
+             --user root \
+             -e GRANT_SUDO=yes \
+             -p 8888:8888  \
+             -v $PWD/tutorials:/home/jovyan/tutorials  \
+             --name workflows-workshop \
+             workflows-workshop:latest
+  ```
+    
+More information about the command line options: https://jupyter-docker-stacks.readthedocs.io/en/latest/using/common.html#notebook-options
+
+
+## Running the notebook with mongodb server
+
+If the `workflows-workshop:latest` image is available you can run the following command to start the tutorial:
+```bash
+docker-compose up
+```
+
+
+## Continuous integration (for future usage)
+
+Each commit triggers a build process on GitLab Runner. Besides the latest tag, there will be a unique tag (same that as the value of the git commit) available for explicitly tracking the version of the notebook for cluster deployment.
+
+### Using the docker image from the registry
+1. Install docker on your machine
+2. Login to the image repository
+    ```bash
+    docker login gitlab-registry.mpcdf.mpg.de 
+    ```
+3. Pull the image:
+   ```bash
+   docker pull gitlab-registry.mpcdf.mpg.de/nomad-lab/analytics-workshop:<tag> 
+   ```
+   
+4. Create a container:
+   ```bash
+   docker run -p 8888:8888 \
+              -v $PWD/notebook/tutorials:/home/jovyan/tutorials \
+              gitlab-registry.mpcdf.mpg.de/nomad-lab/analytics-workshop:<tag>
+   ```
+
+Note: The latest tag can be found on the following page:
+https://gitlab.mpcdf.mpg.de/nomad-lab/analytics-workshop/container_registry
 

configs/abinit/abinit_config.ac9

+# MPI settings
+with_mpi="yes"
+enable_mpi_io="yes"
+
+# linear algebra settings
+with_linalg_flavor="netlib"
+LINALG_LIBS="-L/usr/lib/x86_64-linux-gnu -llapack -lblas"
+
+# mandatory libraries
+with_libxc="yes"
+with_hdf5="yes"
+with_netcdf="yes"
+with_netcdf_fortran="yes"
+
+# fast fourier settings
+with_fft_flavor="fftw3"
+FFTW3_CPPFLAGS="-I/usr/include"
+FFTW3_FCFLAGS="-I/usr/include"
+FFTW3_LIBS="-L/usr/lib/x86_64-linux-gnu -lfftw3 -lfftw3f"
+
+# XML2 library (used by multibinit)
+with_libxml2="yes"
+LIBXML2_FCFLAGS="-I/usr/lib/x86_64-linux-gnu/"
+LIBXML2_LIBS="-L/usr/include/libxml2/libxml/ -lxmlf90"
+

configs/abipy/fw_manager.yaml

+fw_policy:
+    autoparal: True
+    rerun_same_dir: False
+    allow_local_restart: False
+    mpirun_cmd: mpirun
+    max_restarts: 30
+    abipy_manager: /home/jovyan/.abinit/abipy/manager.yml
+

configs/abipy/manager.yml

+
+## Local runner ##############
+
+# policy:
+#   autoparal: 0
+
+hardware: &hardware
+  num_nodes: 1
+  sockets_per_node: 1
+  cores_per_socket: 4
+  mem_per_node: 16 Gb
+
+job: &job
+  mpi_runner: "mpirun"
+  # pre_run: "export PATH=$HOME/git_repos/abinit/build_gcc/src/98_main:$PATH"
+
+
+qadapters: # List of `qadapters` objects  (just one in this simplified example)
+  - priority: 2
+    queue:
+      qtype: shell        # "Submit" jobs via the shell.
+      qname: localhost    # "Submit" to the localhost queue
+                        # (it's a fake queue in this case)
+
+    limits:
+      timelimit: 01:00:00   #  Time-limit for each task.
+      max_cores: 4          #  Max number of cores that can be used by a single task.
+      hint_cores:  2
+
+    hardware: *hardware
+    job: *job
+
+
+  - priority: 1
+    queue:
+      qname: debug
+      qtype: slurm
+    limits:
+      timelimit: 2:00:0
+      min_cores: 1
+      max_cores: 2
+      min_mem_per_proc: 1
+    hardware: *hardware
+    job: *job
+
+
+##############################

configs/abipy/scheduler.yml

+seconds: 30   # number of seconds to wait.
+#minutes: 0  # number of minutes to wait.
+#hours: 0    # number of hours to wait.
+
+
+# Limit on the number of jobs that can be present in the queue. (DEFAULT: 200)
+max_njobs_inqueue: 8
+
+# Maximum number of cores that can be used by the scheduler.
+max_ncores_used: 128

configs/fireworks/FW_config.yaml

+PRINT_FW_JSON: True
+LAUNCHPAD_LOC: /home/jovyan/.fireworks/my_launchpad.yaml
+FWORKER_LOC: /home/jovyan/.fireworks/my_fworker.yaml
+QUEUEADAPTER_LOC: /home/jovyan/.fireworks/my_qadapter.yaml
+SUBMIT_SCRIPT_NAME: FW_submit.script
+QUEUE_UPDATE_INTERVAL: 3
+EXCEPT_DETAILS_ON_RERUN: True
+REMOVE_USELESS_DIRS: True

configs/fireworks/SLURM_template.txt

+#!/bin/bash -l
+
+#SBATCH --nodes=$${nodes}
+#SBATCH --ntasks=$${ntasks}
+#SBATCH --ntasks-per-node=$${ntasks_per_node}
+#SBATCH --ntasks-per-core=$${ntasks_per_core}
+#SBATCH --core-spec=$${core_spec}
+#SBATCH --exclude=$${exclude_nodes}
+#SBATCH --cpus-per-task=$${cpus_per_task}
+#SBATCH --gpus-per-task=$${gpus_per_task}
+#SBATCH --gres=$${gres}
+#SBATCH --qos=$${qos}
+#SBATCH --time=$${walltime}
+#SBATCH --time-min=$${time_min}
+#SBATCH --partition=$${queue}
+#SBATCH --account=$${account}
+#SBATCH --job-name=$${job_name}
+#SBATCH --license=$${license}
+#SBATCH --output=$${job_name}-%j.out
+#SBATCH --error=$${job_name}-%j.error
+#SBATCH --constraint=$${constraint}
+#SBATCH --signal=$${signal}
+#SBATCH --mem=$${mem}
+#SBATCH --mem-per-cpu=$${mem_per_cpu}
+#SBATCH --mail-type=$${mail_type}
+#SBATCH --mail-user=$${mail_user}
+
+
+$${pre_rocket}
+cd $${launch_dir}
+$${rocket_launch}
+$${post_rocket}
+

configs/fireworks/my_fworker.yaml

+name: jovyan_worker
+category: ''
+query: '{}'

configs/fireworks/my_launchpad.yaml

+authsource: fireworks
+host: mongo 
+logdir: null
+mongoclient_kwargs: {}
+name: fireworks
+password: null
+port: 27017
+ssl: false
+ssl_ca_certs: null
+ssl_certfile: null
+ssl_keyfile: null
+ssl_pem_passphrase: null
+strm_lvl: INFO
+uri_mode: false
+user_indices: []
+username: null
+wf_user_indices: []

configs/fireworks/my_qadapter.yaml

+_fw_name: CommonAdapter
+_fw_q_type: SLURM
+_fw_template_file: /home/jovyan/.fireworks/SLURM_template.txt
+rocket_launch: rlaunch singleshot
+ntasks: 1
+walltime: '02:0:0'
+queue: debug
+account: null
+job_name: null
+pre_rocket: null
+post_rocket: null

configs/slurm/slurm.conf

+# slurm.conf file generated by configurator easy.html.
+# Put this file on all nodes of your cluster.
+# See the slurm.conf man page for more information.
+
+ControlMachine=localhost
+ControlAddr=127.0.0.1
+
+# MailProg=/bin/mail
+MpiDefault=none
+# MpiParams=ports=#-#
+ProctrackType=proctrack/pgid
+ReturnToService=1
+SlurmctldPidFile=/var/run/slurm-llnl/slurmctld.pid
+# SlurmctldPort=6817
+SlurmdPidFile=/var/run/slurm-llnl/slurmd.pid
+# SlurmdPort=6818
+SlurmdSpoolDir=/var/lib/slurm-llnl/slurmd
+
+SlurmUser=jovyan
+SlurmdUser=jovyan
+StateSaveLocation=/var/lib/slurm-llnl/slurmctld
+SwitchType=switch/none
+TaskPlugin=task/none
+
+# TIMERS
+# KillWait=30
+# MinJobAge=300
+# SlurmctldTimeout=120
+# SlurmdTimeout=300
+
+# SCHEDULING
+FastSchedule=1
+SchedulerType=sched/backfill
+# SchedulerPort=7321
+SelectType=select/linear
+
+# LOGGING AND ACCOUNTING
+AccountingStorageType=accounting_storage/none
+ClusterName=cluster
+# JobAcctGatherFrequency=30
+JobAcctGatherType=jobacct_gather/none
+# SlurmctldDebug=3
+SlurmctldLogFile=/var/log/slurm-llnl/slurmctld.log
+# SlurmdDebug=3