Merge pull request #1 from modl-uclouvain/dev

Dev

.dockerignore

+**/.git
+
+# mongodb instance folder
+db/
+
+# results of the the notebooks
+tutorials/flow_sic_relax
+tutorials/launcher_*/
+

.gitignore

@@ -5,7 +5,35 @@ db/
 tutorials/flow_sic_relax
 tutorials/launcher_*/
 
-
+# https://github.com/github/gitignore/blob/master/Global/macOS.gitignore
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# https://github.com/github/gitignore/blob/master/Python.gitignore
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -28,7 +56,6 @@ parts/
 sdist/
 var/
 wheels/
-pip-wheel-metadata/
 share/python-wheels/
 *.egg-info/
 .installed.cfg
@@ -58,6 +85,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+cover/
 
 # Translations
 *.mo
@@ -80,6 +108,7 @@ instance/
 docs/_build/
 
 # PyBuilder
+.pybuilder/
 target/
 
 # Jupyter Notebook
@@ -90,7 +119,9 @@ profile_default/
 ipython_config.py
 
 # pyenv
-.python-version
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
@@ -135,3 +166,9 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
\ No newline at end of file

Dockerfile

 # https://github.com/jupyter/docker-stacks
 # https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html
-FROM jupyter/scipy-notebook
-LABEL maintainer="Gian-Marco Rignanese <gian-marco.rignanese@uclouvain.be>"
 
+FROM jupyter/scipy-notebook AS builder
 
-# # Fix DL4006
-# SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-# Abinit installation
-# ===================
+# Abinit compilation
+# ==================
 # # 1. compiler: gfortran
 # # 2. MPI libraries - choice for Open MPI: mpi-default-dev libopenmpi-dev
 # # 3. math libraries - choice for lapack and blas: liblapack-dev libblas-dev
@@ -23,57 +19,89 @@ RUN apt-get update \
     liblapack-dev libblas-dev \
     libhdf5-dev libnetcdf-dev libnetcdff-dev libpnetcdf-dev libxc-dev \
     libfftw3-dev libxml2-dev \
-    slurmd slurm-client slurmctld \
  && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /opt
+WORKDIR /tmp
 
 ARG abinit_version="9.4.0"
-COPY configs/abinit_config.ac9 /opt/abinit-${abinit_version}/build/abinit_config.ac9
+COPY configs/abinit_config.ac9 /tmp/abinit-${abinit_version}/build/abinit_config.ac9
 
 RUN wget -k "https://www.abinit.org/sites/default/files/packages/abinit-${abinit_version}.tar.gz" \
  && tar xzf abinit-${abinit_version}.tar.gz \
  && cd abinit-${abinit_version} \
  && cd build \
- && ../configure --with-config-file='abinit_config.ac9'   \
+ && ../configure --with-config-file='abinit_config.ac9'  --prefix=/opt/abinit \
  && make -j4 \
- && make install \
- && rm /opt/abinit-${abinit_version}.tar.gz \
- # && rm -rf /opt/abinit-${abinit_version}
- && fix-permissions "/opt/abinit-${abinit_version}"
+ && make install
+
+
+FROM jupyter/scipy-notebook
+LABEL maintainer="Gian-Marco Rignanese <gian-marco.rignanese@uclouvain.be>"
+
+# Abinit installation
+# ===================
+# # 1. MPI libraries - choice for Open MPI: mpi-default-bin openmpi-bin libopenmpi3
+# # 2. math libraries - choice for lapack and blas: liblapack3 libblas3
+# # 3. mandatory libraries: libhdf5-103 libnetcdf15 libnetcdff7 libpnetcdf0d libxc5 libfftw3-bin libxml2
+
+USER root
+
+RUN apt-get update \
+ && apt install -y --no-install-recommends \
+    mpi-default-bin openmpi-bin libopenmpi3 \
+    liblapack3 libblas3 \
+    libhdf5-103 libnetcdf15 libnetcdff7 libpnetcdf0d libxc5 \
+    libfftw3-bin libxml2 \
+ && rm -rf /var/lib/apt/lists/*
 
 USER $NB_UID
 
+COPY --chown=$NB_UID:$NB_GID --from=builder /opt/abinit /opt/abinit
+ENV PATH=/opt/abinit/bin:$PATH
+
+
 # Install Python 3 packages
 # =========================
+# fireworks's depenecies: flask-paginate gunicorn pymongo
+# pseudo_dojo's depenecies: periodic-table-plotter atomicfile
 
 RUN conda install --quiet --yes \
-    # 'ase' \
-    # 'pymatgen' \
-    # 'atomate' \
-    # 'matminer' \
-    # 'jupyterlab-git' \
     'abipy' \
     'jupyter-server-proxy' \
-    'jupyter_contrib_nbextensions' \
-    'jupyter_nbextensions_configurator' \
+    'flask-paginate' 'gunicorn' 'pymongo' \
+    'periodic-table-plotter' 'atomicfile' \
  && pip install --no-cache-dir jupyter-jsmol fireworks \
  && conda clean --all -f -y \
  && fix-permissions "${CONDA_DIR}" \
  && fix-permissions "/home/${NB_USER}"
 
 # Pseudo-dojo
+# ===========
+
+COPY --chown=$NB_UID:$NB_GID pseudo_dojo /opt/pseudo_dojo
+WORKDIR /opt/pseudo_dojo
+RUN pip install -e .
+
+# Setup slurm
+# ===========
+
 USER root
-WORKDIR /opt
 
-RUN git clone --depth 1 https://github.com/abinit/pseudo_dojo.git \
- && cd pseudo_dojo \
- && fix-permissions "/opt/pseudo_dojo"
+RUN apt-get update \
+ && apt install -y --no-install-recommends \
+    slurmd slurm-client slurmctld \
+ && rm -rf /var/lib/apt/lists/*
 
-USER $NB_UID
+COPY configs/slurm.conf /etc/slurm-llnl/slurm.conf
+COPY configs/start_slurm.sh  /usr/local/bin/before-notebook.d/
 
-RUN pip install -e pseudo_dojo
+# https://github.com/yuwata/slurm-fedora/blob/master/slurm-setuser.in
+RUN mkdir -p /run/munge \
+ && chown -R jovyan /run/munge /etc/munge /var/lib/munge /var/log/munge \
+ && mkdir -p /var/run/slurm-llnl \
+ && chown -R jovyan /var/run/slurm-llnl /var/lib/slurm-llnl /var/log/slurm-llnl
 
 USER $NB_UID
 WORKDIR $HOME
 
+COPY --chown=$NB_UID:$NB_GID tutorials tutorials

configs/slurm.conf

+# slurm.conf file generated by configurator easy.html.
+# Put this file on all nodes of your cluster.
+# See the slurm.conf man page for more information.
+
+ControlMachine=localhost
+ControlAddr=127.0.0.1
+
+# MailProg=/bin/mail
+MpiDefault=none
+# MpiParams=ports=#-#
+ProctrackType=proctrack/pgid
+ReturnToService=1
+SlurmctldPidFile=/var/run/slurm-llnl/slurmctld.pid
+# SlurmctldPort=6817
+SlurmdPidFile=/var/run/slurm-llnl/slurmd.pid
+# SlurmdPort=6818
+SlurmdSpoolDir=/var/lib/slurm-llnl/slurmd
+
+SlurmUser=jovyan
+SlurmdUser=jovyan
+StateSaveLocation=/var/lib/slurm-llnl/slurmctld
+SwitchType=switch/none
+TaskPlugin=task/none
+
+# TIMERS
+# KillWait=30
+# MinJobAge=300
+# SlurmctldTimeout=120
+# SlurmdTimeout=300
+
+# SCHEDULING
+FastSchedule=1
+SchedulerType=sched/backfill
+# SchedulerPort=7321
+SelectType=select/linear
+
+# LOGGING AND ACCOUNTING
+AccountingStorageType=accounting_storage/none
+ClusterName=cluster
+# JobAcctGatherFrequency=30
+JobAcctGatherType=jobacct_gather/none
+# SlurmctldDebug=3
+SlurmctldLogFile=/var/log/slurm-llnl/slurmctld.log
+# SlurmdDebug=3
+SlurmdLogFile=/var/log/slurm-llnl/slurmd.log
+
+# COMPUTE NODES
+NodeName=localhost CPUs=1 State=UNKNOWN
+PartitionName=debug Nodes=localhost Default=YES MaxTime=INFINITE State=UP

configs/start_slurm.sh

+#!/bin/bash
+
+# Handle special flags if we're root
+if [ $(id -u) == 0 ]; then
+    su jovyan /etc/init.d/munge start
+    su jovyan /etc/init.d/slurmd start
+    su jovyan /etc/init.d/slurmctld start
+else
+    /etc/init.d/munge start
+    /etc/init.d/slurmd start
+    /etc/init.d/slurmctld start
+fi
\ No newline at end of file

pseudo_dojo/INSTALL

+.. The source of this document is INSTALL. During the doc build process,
+.. this file is copied over to doc/users/installing.rst.
+.. Therefore, you must edit INSTALL, *not* doc/users/installing.rst!
+
+**********
+Installing
+**********
+
+.. _install_requirements:
+
+Build requirements
+==================
+
+These are external packages which you will need to install before installing pseudo_dojo. 
+
+:term:`python` 2.7 (or later but not python3)
+
+:term:`numpy` 1.1 (or later)
+    array support for python 
+    (`download <http://sourceforge.net/project/showfiles.php?group_id=1369&package_id=175103>`__)
+
+:term:`scipy`  
+
+:term:`matplotlib` 
+
+.. _install_from_source:
+
+Installing from source
+======================
+
+Once you have satisfied the requirements detailed above, you can build pseudo_dojo::
+
+  cd pseudo_dojo
+  python setup.py build
+  python setup.py install
+
+

pseudo_dojo/LICENSE

+pseudo_dojo is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 2.1 of the License, or
+(at your option) any later version.
+
+pseudo_dojo is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along with pseudo_dojo.  
+If not, see <http://www.gnu.org/licenses/>.

pseudo_dojo/README.md

+
+PseudoDojo is an open-source Python framework for generating and validating pseudopotentials.
+It uses the [AbiPy](https://github.com/abinit/abipy) package, for developing and systematically 
+testing pseudopotentials. 
+At present it contains seven different batteries of tests executed with [ABINIT](https://github.com/abinit/abinit) 
+([delta-gauge](https://molmod.ugent.be/deltacodesdft), 
+revised delta gauge by [Jollet et al](http://www.sciencedirect.com/science/article/pii/S0010465513004359), 
+[GBRV](https://www.physics.rutgers.edu/gbrv/) tests for fcc/bcc/compounds, 
+phonons at the Gamma point and tests for the presence of ghost-states below and above the Fermi level).
+
+The tests are performed as a function of the energy cutoff (with the exception of ghosts and compounds) 
+and the results are then used to provide hints for the energy cutoff for actual production calculations. 
+We keep track of the results of the various validation tests for each pseudopotential with the `djrepo` file, 
+a text document in JSON format produced by the python code at the end of the test. 
+For further details, please consult our recent [cond-mat paper](https://arxiv.org/abs/1710.10138).
+
+The PseudoDojo code is hosted on [github](https://github.com/abinit/pseudo_dojo) 
+but we also provide a user web-interface at <http://www.pseudo-dojo.org>
+There you will find pseudopotential files that can be used immediately, 
+as well as the corresponding input files if you need to change or tune or change some parameters 
+(e.g. the XC functional).
+
+The pseudopotential files are available on the web-site in the ABINIT `psp8` format, 
+in the `UPF2` format and in the `PSML` 1.1 XML format shared by SIESTA and ABINIT. 
+The input files, the results of the generation, and the test results are presented via jupyter notebooks 
+as static HTML pages. 
+One can hence easily compare pseudopotentials for a given element and then select the most appropriate 
+one according to a chosen criterion (e.g. efficiency vs accuracy).  
+
+How to cite the PseudoDojo project
+----------------------------------
+
+If you use the PseudoDojo pseudopotentials in your research, 
+please consider citing the works mentioned in [this page](http://www.pseudo-dojo.org/faq.html).
+
+Getting PseudoDojo
+------------------
+
+Developmental version
+---------------------
+
+The developmental version is at the PseudoDojo's `Github repo <https://github.com/gmatteo/pseudo_dojo>`_. 
+After cloning the source, you can type::
+
+    python setup.py install
+
+or, alternatively,
+
+    python setup.py develop
+
+to install the package in developmental mode.
+
+<!--
+Stable version
+==============
+
+The version at the Python Package Index (PyPI) is always the latest stable
+release that will be hopefully, be relatively bug-free. The easiest way to
+install PseudoDojo is to use pip, as follows::
+
+    pip install pseudo_dojo
+-->
+
+
+Project PseudoDojo: global view
+===============================
+
+Global long-term objectives:
+----------------------------
+
+- To have, on the Web, sets of validated pseudopotentials, for the whole periodic table,
+  for different exchange-correlation functionals, with different possibilities of 
+  semi-core electrons (e.g. for GW), different cut-off radii (e.g. high pressure application), 
+  with an optimal cut-off energy.
+
+- To have a Web portal for the generation/validation of new pseudopotentials.
+
+- Computation of selected physical properties for selected systems, associated with one 
+  given pseudopotential (automatic computation of the cut-off energy, computation of the 
+  total energy, the interatomic distance, the lattice parameter of the elemental solid 
+  and one oxide, also dimer). Results presented on the Web.
+
+- Validation of pseudopotentials with respect to a reference.
+
+License
+=======
+
+The pseudopotential files as well as the input files are released under the 
+[CC BY 4.0 license](https://creativecommons.org/licenses/by/4.0/legalcode)
+This license lets you distribute, remix, tweak, and build upon our work, even commercially, 
+as long as you credit the PseudoDojo project for the original creation. 
+
+The PseudoDojo code (the python code used to generate/validate the pseudos) 
+is released under the GPL License. 
+The terms of the license are as follows:
+
+PseudoDojo is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 2.1 of the License, or
+(at your option) any later version.
+
+PseudoDojo is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along with PseudoDojo.  
+If not, see <http://www.gnu.org/licenses/>.

pseudo_dojo/pseudo_dojo/__init__.py

+from __future__ import division, print_function, unicode_literals
+
+import os
+
+from collections import OrderedDict
+from collections import abc
+from monty.dev import deprecated
+from monty.functools import lazy_property
+from monty.design_patterns import singleton
+#from pymatgen.core.periodic_table import Element
+from pseudo_dojo.core.pseudos import OfficialDojoTable
+from pseudo_dojo.pseudos import dojotable_absdir
+
+
+@deprecated(message="Official PseudoDojo API will be released soon!")
+def get_pseudos(top):
+    """
+    Find pseudos within top, return :class:`PseudoTable` object sorted by atomic number Z.
+    """
+    from monty.os.path import find_exts
+    from pymatgen.io.abinit.pseudos import PseudoTable, Pseudo
+    exts = ("psp8",)
+    pseudos = []
+    for p in find_exts(top, exts, exclude_dirs="_*"):
+        try:
+            pseudos.append(Pseudo.from_file(p))
+        except Exception as exc:
+            from warnings import warn
+            warn("Exception in pseudo %s:\n%s" % (p.filepath, exc))
+
+    return PseudoTable(pseudos).sort_by_z()
+
+
+class TableMetadata(object):
+    """
+    Metadata associated to one of the official `PseudoDojo` tables.
+    """
+    def __init__(self, table_dir, djson_name):
+        """
+        Args:
+            table_dir: basename of the directory containing the pseudos
+            djson_name: name of the json file in `table_dir` with the
+                list of pseudos and metatada.
+        """
+        self.table_dir = table_dir
+        self.dojo_absdir = dojotable_absdir(table_dir)
+        self.djson_name = djson_name
+        self.djson_path = os.path.join(self.dojo_absdir, djson_name)
+
+
+@singleton
+class OfficialTables(abc.Mapping):
+    """
+    Official tables provided by the PseudoDojo project.
+
+    Naming scheme:
+
+        [PP_TYPE]-[XC_NAME]-[TABLE_NAME_WITH_VERSION]-[DJSON_NAME]
+
+    where:
+
+        #. PP_TYPE in ("PAW", "NC") defines the pseudos type.
+        #. XC_NAME defines the exchange-correlation functional e.g. PBE
+        #. TABLE_NAME_WITH_VERSION is the name of the table e.g. ONCVPSP, JTH ...
+        #. DJSON_NAME is the name of the djson file located in the pseudodojo directory.
+
+    In the case of NC pseudos, the SOC term is treated via djson e.g. "accuracy-soc.djson".
+    maybe FR|SR for fully or scalar-relativistic?
+    """
+    def __init__(self):
+        self._tables = tables = OrderedDict()
+
+        #tables["ONCVPSP-PBE-PDv0.2-accuracy"] = TableMetadata("ONCVPSP-PBE-PDv0.2", "test_accuracy.djson")
+        tables["ONCVPSP-PBE-PDv0.2-accuracy"] = TableMetadata("ONCVPSP-PBE-PDv0.3", "standard.djson")
+        #tables["ONCV-GYGYv0.2-PBE"] = TableMetadata("ONC-PBE-GYGYv0.2", "htc.djson")
+        #tables["PAW-JTHv0.2-PBE"] = TableMetadata("PAW-PBE-JTHv0.2", "standard.djson")
+        #tables["PAW-PBE-GBRVv0.2"] = TableMetadata("PAW-PBE-GBRVv0.2", "efficiency.djson")
+        # TODO: Add check on md5 of djson files.
+
+    # ABC protocol.
+    def __iter__(self):
+        return self._tables.__iter__()
+
+    def __len__(self):
+        return self._tables.__len__()
+
+    def __getitem__(self, key):
+        """Called by self[key]"""
+        v = self._tables[key]
+
+        # Return v if v is table else parse the files, construct table and store it.
+        if not isinstance(v, TableMetadata):
+            return v
+
+        new_table = OfficialDojoTable.from_djson_file(v.djson_path)
+        new_table.dojo_name = key
+        self._tables[key] = new_table
+
+        return new_table
+
+    def select_tables(self, pp_type=None, xc=None):
+        """
+        Low-level method used to select tables.
+
+        Args:
+            pp_type: NC for norm-conserving pseudos or PAW. If None no selection is done.
+            xc: If xc is not None, only the pseudos with this xc functional are selected.
+                else no selection is performed.
+        Return:
+            List of tables.
+        """
+        tables = self.values()
+        if pp_type is not None:
+            tables = [t for t in tables if t.dojo_info.pp_type == pp_type]
+        if xc is not None:
+            tables = [t for t in tables if t.dojo_info.xc == xc]
+
+        return tables
+
+    @lazy_property
+    def available_xcs(self):
+        """List with the XC functionals available."""
+        return sorted(set(t.xc for t in self.values()))
+
+    def all_nctables(self, xc=None):
+        """
+        Return all norm-conserving tables available.
+        If xc is not None, only the pseudos with this xc functional are selected.
+        """
+        tables = [t for t in self.values() if t.dojo_info.isnc]
+        if xc is not None:
+            tables = [t for t in tables if t.xc == xc]
+        return tables
+
+    def all_pawtables(self, xc=None):
+        """
+        Return all PAW tables available.
+        If xc is not None, only the pseudos with this xc functional are selected.
+        """
+        tables = [table for table in self.values() if table.dojo_info.ispaw]
+        if xc is not None:
+            tables = [t for t in tables if t.xc == xc]
+        return tables
+
+    def pseudo_from_symbol_md5(self, symbol, md5, pp_type, xc):
+        """
+        Find pseudo from chemical symbol, md5 checksum, pp_type and xc.
+
+        Raises:
+            ValueError if pseudo is not found.
+        """
+        # This is the __eq__ implemented for Pseudo
+        #return (self.md5 == other.md5 and
+        #        self.__class__ == other.__class__ and
+        #        self.Z == other.Z and
+        #        self.Z_val == other.Z_val and
+        #        self.l_max == other.l_max )
+
+        # Here it's very important the we don't have duplicated md5 in the pseudo dojo tables.
+        # Actually there's a check in test/test_dojotables.py
+        for p in self.select_pseudos(symbol, pp_type=pp_type, xc=xc):
+            if p.md5 == md5: return p
+
+        raise ValueError(
+            "Cannot find pseudo associated to the following parameters:\n"
+            "symbol=%s, md5=%s, pp_type=%s, xc=%s" % (symbol, md5, pp_type, xc))
+
+    def select_pseudos(self, symbol, pp_type=None, xc=None):
+        """
+        Return the full list of Pseudo objects available in the DojoTables
+        with the given `pp_type` and XC functional `xc`.
+        """
+        tables = self.select_tables(pp_type=pp_type, xc=xc)
+        pseudos = []
+        for tab in tables:
+            pseudos.extend(tab.pseudo_with_symbol(symbol, allow_multi=True))
+        return pseudos
+
+    def select_nc_pseudos(self, symbol, xc=None):
+        """
+        Return list of NC pseudos with the given chemical `symbol`.
+        If xc is not None, only the pseudos with this xc functional are selected.
+        """
+        return self.select_pseudos(symbol, pp_type="NC", xc=xc)
+
+    def select_paw_pseudos(self, symbol, xc=None):
+        """
+        Return list of PAW pseudos with the given chemical `symbol`.
+        If xc is not None, only the pseudos with this xc are selected.
+        """
+        return self.select_pseudos(symbol, pp_type="PAW", xc=xc)