setting up mongodb

.gitignore

+# mongodb instance folder
+db/
+
+# results of the the notebooks
+tutorials/flow_sic_relax
+tutorials/launcher_*/
+
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

Dockerfile

@@ -23,7 +23,6 @@ RUN apt-get update \
     liblapack-dev libblas-dev \
     libhdf5-dev libnetcdf-dev libnetcdff-dev libpnetcdf-dev libxc-dev \
     libfftw3-dev libxml2-dev \
-    mongodb \
     slurmd slurm-client slurmctld \
  && rm -rf /var/lib/apt/lists/*
 
@@ -55,6 +54,7 @@ RUN conda install --quiet --yes \
     # 'matminer' \
     # 'jupyterlab-git' \
     'abipy' \
+    'jupyter-server-proxy' \
     'jupyter_contrib_nbextensions' \
     'jupyter_nbextensions_configurator' \
  && pip install --no-cache-dir jupyter-jsmol fireworks \
@@ -74,6 +74,6 @@ USER $NB_UID
 
 RUN pip install -e pseudo_dojo
 
-
+USER $NB_UID
 WORKDIR $HOME
 

README.md

@@ -78,6 +78,14 @@ More info: https://jupyter-docker-stacks.readthedocs.io/en/latest/using/common.h
 More information about the command line options: https://jupyter-docker-stacks.readthedocs.io/en/latest/using/common.html#notebook-options
 
 
+## Running the notebook with mongodb server
+
+If the `workflows-workshop:latest` image is available you can run the following command to start the tutorial:
+```bash
+docker-compose up
+```
+
+
 ## Continuous integration (for future usage)
 
 Each commit triggers a build process on GitLab Runner. Besides the latest tag, there will be a unique tag (same that as the value of the git commit) available for explicitly tracking the version of the notebook for cluster deployment.

docker-compose.yml

+version: '3.1'
+services:
+
+  workflows-workshop:
+    # build: .
+    image: workflows-workshop:latest
+    restart: always
+    user: root
+    environment:
+      # - NB_UID=1001
+      - GRANT_SUDO=yes
+    ports:
+      - 8888:8888
+    depends_on:
+      - mongo
+    volumes:
+      - ./tutorials:/home/jovyan/tutorials
+
+  mongo:
+    image: mongo
+    restart: always
+    # environment:
+    #   - MONGO_INITDB_ROOT_USERNAME=admin
+    #   - MONGO_INITDB_ROOT_PASSWORD=secret
+    # ports:
+    #   - 27017:27017  # opening port for development only!
+    volumes:
+      - ./db:/data/db
+
+  # mongo-express:  # mongo-express only for development only!
+  #   image: mongo-express
+  #   restart: always
+  #   ports:
+  #     - 8081:8081
+  #   environment:
+  #     - ME_CONFIG_MONGODB_ADMINUSERNAME=admin
+  #     - ME_CONFIG_MONGODB_ADMINPASSWORD=secret
+  #   depends_on:
+  #     - mongo

tutorials/Abinit - quickstart.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "exempt-cloud",
+   "metadata": {},
+   "source": [
+    "# Abinit - quickstart\n",
+    "\n",
+    "https://abinit.github.io/abipy/flow_gallery/run_sic_relax.html#sphx-glr-flow-gallery-run-sic-relax-py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "necessary-budapest",
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "!abicheck.py"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "chronic-invention",
+   "metadata": {},
+   "source": [
+    "## Relaxation Flow\n",
+    "\n",
+    "This example shows how to build a very simple Flow for the structural relaxation of SiC. One could use a similar logic to perform multiple relaxations with different input parameters…\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "handmade-dividend",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import abipy.abilab as abilab\n",
+    "import abipy.data as data\n",
+    "import abipy.flowtk as flowtk\n",
+    "\n",
+    "\n",
+    "def build_flow(workdir):\n",
+    "\n",
+    "    pseudos = data.pseudos(\"14si.pspnc\", \"6c.pspnc\")\n",
+    "    structure = data.structure_from_ucell(\"SiC\")\n",
+    "\n",
+    "    # Initialize the input\n",
+    "    relax_inp = abilab.AbinitInput(structure, pseudos=pseudos)\n",
+    "\n",
+    "    # Set variables\n",
+    "    relax_inp.set_vars(\n",
+    "        ecut=20,\n",
+    "        paral_kgb=1,\n",
+    "        iomode=3,\n",
+    "        # Relaxation part\n",
+    "        ionmov=2,\n",
+    "        optcell=1,\n",
+    "        strfact=100,\n",
+    "        ecutsm=0.5,       # Important!\n",
+    "        dilatmx=1.15,     # Important!\n",
+    "        toldff=1e-6,\n",
+    "        tolmxf=1e-5,\n",
+    "        ntime=100,\n",
+    "    )\n",
+    "\n",
+    "    # K-points sampling\n",
+    "    shiftk = [\n",
+    "        [0.5,0.5,0.5],\n",
+    "        [0.5,0.0,0.0],\n",
+    "        [0.0,0.5,0.0],\n",
+    "        [0.0,0.0,0.5]\n",
+    "    ]\n",
+    "    relax_inp.set_kmesh(ngkpt=[4, 4, 4], shiftk=shiftk)\n",
+    "\n",
+    "    # Initialize the flow\n",
+    "    flow = flowtk.Flow(workdir)\n",
+    "\n",
+    "    # Register the task.\n",
+    "    flow.register_relax_task(relax_inp)\n",
+    "\n",
+    "    return flow"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "related-music",
+   "metadata": {},
+   "source": [
+    "Build and run the flow:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "steady-fellowship",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "flow = build_flow('flow_sic_relax')\n",
+    "\n",
+    "scheduler = flow.make_scheduler()\n",
+    "scheduler.start()\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "confirmed-space",
+   "metadata": {},
+   "source": [
+    "To visualize the evolution of the lattice parameters during the structura relaxation use:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "raising-valuable",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "abifile = abilab.abiopen('flow_sic_relax/w0/t0/outdata/out_HIST.nc')\n",
+    "abifile.plot();"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "changing-bennett",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "passing-mineral",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "prescribed-mexico",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

tutorials/Fireworks - quickstart.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "metallic-supervisor",
+   "metadata": {},
+   "source": [
+    "# Fireworks (Five-minute quickstart)\n",
+    "\n",
+    "https://materialsproject.github.io/fireworks/quickstart.html"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "gorgeous-curve",
+   "metadata": {},
+   "source": [
+    "## Start FireWorks\n",
+    "\n",
+    "If not already running, start MongoDB (if your MongoDB is hosted and maintained externally, follow the note below regarding lpad init):\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "electronic-essex",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# !lpad init "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "white-owner",
+   "metadata": {},
+   "source": [
+    "Reset/Initialize the FireWorks database (the LaunchPad):"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "entertaining-scotland",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "# !lpad reset "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "increasing-context",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!cat my_launchpad.yaml"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "numerous-regard",
+   "metadata": {},
+   "source": [
+    "## Add a Workflow\n",
+    "\n",
+    "There are many ways to add Workflows to the database, including a Python API. Let’s start with an extremely simple example that can be added via the command line:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "square-primary",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!lpad add_scripts 'echo \"hello\"' 'echo \"goodbye\"' -n hello goodbye -w test_workflow"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "manufactured-bernard",
+   "metadata": {},
+   "source": [
+    "This added a two-job linear workflow. The first jobs prints hello to the command line, and the second job prints goodbye. We gave names (optional) to each step as “hello” and “goodbye”. We named the workflow overall (optional) as “test_workflow”."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "mounted-audio",
+   "metadata": {},
+   "source": [
+    "Let’s look at our test workflow:\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "identical-international",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!lpad get_wflows -n test_workflow -d more"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fuzzy-intellectual",
+   "metadata": {},
+   "source": [
+    "We get back basic information on our workflows. The second step “goodbye” is waiting for the first one to complete; it is not ready to run because it depends on the first job.\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "useful-reviewer",
+   "metadata": {},
+   "source": [
+    "## Run all Workflows\n",
+    "\n",
+    "You can run jobs one at a time (“singleshot”) or all at once (“rapidfire”). Let’s run all jobs:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "excess-chosen",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!rlaunch --silencer rapidfire"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "dedicated-commerce",
+   "metadata": {},
+   "source": [
+    "Clearly, both steps of our workflow ran in the correct order."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "joined-antique",
+   "metadata": {},
+   "source": [
+    "Let’s again look at our workflows:\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "conditional-henry",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!lpad get_wflows -n test_workflow -d more"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "lucky-failing",
+   "metadata": {},
+   "source": [
+    "\n",
+    "FireWorks automatically created launcher_ directories for each step in the Workflow and ran them. We see that both steps are complete. Note that there exist options to choose where to run jobs, as well as to tear down empty directories after running jobs.\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "forward-client",
+   "metadata": {},
+   "source": [
+    "## Launch the web GUI\n",
+    "If you have a web browser, you can launch the web GUI to see your results:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "anticipated-patrol",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# !lpad webgui"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "civic-sauce",
+   "metadata": {},
+   "source": [
+    "Note that there are options to run the web site in a server mode, try lpad webgui -h to see all the options.\n",
+    "\n",
+    "open: http://localhost:8888/proxy/5000"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "explicit-denial",
+   "metadata": {},
+   "source": [
+    "## Python code\n",
+    "\n",
+    "The following Python code achieves the same behavior:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cutting-modification",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from fireworks import Firework, Workflow, LaunchPad, ScriptTask\n",
+    "from fireworks.core.rocket_launcher import rapidfire\n",
+    "\n",
+    "# set up the LaunchPad and reset it\n",
+    "launchpad = LaunchPad(host='mongo')\n",
+    "launchpad.reset('', require_password=False)\n",
+    "\n",
+    "# create the individual FireWorks and Workflow\n",
+    "fw1 = Firework(ScriptTask.from_str('echo \"hello\"'), name=\"hello\")\n",
+    "fw2 = Firework(ScriptTask.from_str('echo \"goodbye\"'), name=\"goodbye\")\n",
+    "wf = Workflow([fw1, fw2], {fw1:fw2}, name=\"test workflow\")\n",
+    "\n",
+    "# store workflow and launch it locally\n",
+    "launchpad.add_wf(wf)\n",
+    "rapidfire(launchpad)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "allied-lesson",
+   "metadata": {},
+   "source": [
+    "In the code above, the `{fw1:fw2}` argument to Workflow is adding a dependency of fw2 to fw1. You could instead define this dependency when defining your FireWorks:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "interstate-intervention",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fw1 = Firework(ScriptTask.from_str('echo \"hello\"'), name=\"hello\")\n",
+    "fw2 = Firework(ScriptTask.from_str('echo \"goodbye\"'), name=\"goodbye\", parents=[fw1])\n",
+    "wf = Workflow([fw1, fw2], name=\"test workflow\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "speaking-institution",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "devoted-treat",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

configs/manager.yml → tutorials/manager.yml

@@ -16,14 +16,14 @@ qadapters: # List of `qadapters` objects  (just one in this simplified example)
     # pre_run: "export PATH=$HOME/git_repos/abinit/build_gcc/src/98_main:$PATH"
 
   limits:
-    timelimit: 24:00:00   #  Time-limit for each task.
-    max_cores: 64          #  Max number of cores that can be used by a single task.
-    hint_cores:  8
+    timelimit: 01:00:00   #  Time-limit for each task.
+    max_cores: 4          #  Max number of cores that can be used by a single task.
+    hint_cores:  2
 
   hardware:
     num_nodes: 1
-    sockets_per_node: 2
-    cores_per_socket: 64
-    mem_per_node: 256 Gb
+    sockets_per_node: 1
+    cores_per_socket: 4
+    mem_per_node: 16 Gb
 
 ##############################

tutorials/my_launchpad.yaml

+authsource: fireworks
+host: mongo 
+logdir: null
+mongoclient_kwargs: {}
+name: fireworks
+password: null
+port: 27017
+ssl: false
+ssl_ca_certs: null
+ssl_certfile: null
+ssl_keyfile: null
+ssl_pem_passphrase: null
+strm_lvl: INFO
+uri_mode: false
+user_indices: []
+username: null
+wf_user_indices: []