Merge branch 'master' into 'master'

Added testing inventory

See merge request !52

basic_configuration/give_docker_registry_access.yml

@@ -12,5 +12,3 @@
       copy:
         src: data/registry.crt
         dest: "/etc/docker/certs.d/{{ docker_registry }}:{{ docker_registry_port }}/registry.crt"
-
-

basic_configuration/main.yml

@@ -138,5 +138,3 @@
         - never
         - build
         - buildbase
-
-

effelsberg_testing/group_vars/all.yml

+---
+enable_dhcpd: True
+# Repository Settings
+#####################
+#
+# These settings specify the repositories used to build the EDD
+# In particular useful to be tweaked in a local custom mysite_dev inventory to develop  the provisioning system.
+version_tag: "latest"
+
+# Repository and branch used toc heck out mpikat
+mpikat_repository: "https://gitlab.mpcdf.mpg.de/nesser/mpikat.git"
+mpikat_branch: "dev"
+
+# Repository and branch of the provisiioing system (roles and provision descriptions) used by the master controller
+provision_repository: "https://gitlab.mpcdf.mpg.de/mpifr-bdg/edd_provisioning.git"
+provision_branch: "master"
+
+# Inventory file / directory in this repository to be used.
+# ToDo: replace with look up of the current inventory
+edd_inventory_folder: "effelsberg_testing"
+
+
+# Data path for the docker registry
+docker_registry_data_path: "/beegfsEDD/edd_docker_registry"
+
+# Data path for the influx db database
+influx_db_path: "/beegfsEDD/EDD_pipeline_data/testing/influx_db"
+
+# Architecture for gcc optimizations
+gcc_architecture: "broadwell"   # to use same container on pacifixes as on edds
+
+
+# Tweak to use the correct python version depending on the used ansible version.
+ansible_python_interpreter: auto_legacy_silent
+
+
+# Network configuration
+#######################
+#
+# Parameters of the network configuration of the site
+
+# The subnet from which the katcp server accept control connections
+edd_subnet: "0.0.0.0"
+
+# Subnet used for highspeed data connections
+high_speed_data_subnet: "10.10.1.0"
+high_speed_data_subnetmask: "255.255.255.0"
+
+# Port used for the ssh conenctions of the master controller tot he ansible interface
+edd_ansible_port: 2222
+
+
+
+
+# Variables for Effelsberg specific services
+##############################################
+#
+# These variables are only used for the effelsberg setup.
+log_server: "effgraylog.mpifr-bonn.mpg.de:12202"
+
+fits_interface_port: 5002
+effelsberg_scpi_port: 5025
+
+
+
+# File System Settings
+#########################
+#
+# Settings of the shared file system / common file system layout of the nodes.
+
+# Data base path to be used for pipelines. The pipelines will create
+# data_base_path/container_name as directory and see this internally as /mnt
+# This can be overriden on per role basis by setting data_path variable
+# for the role
+data_base_path: "/beegfsEDD/EDD_pipeline_data/testing"
+
+# base path as used for additional data definitions
+# ToDo: Move to role default variables
+base_path: "{{ playbook_dir }}"
+
+
+
+# Service variables
+#####################
+#
+# These variables are used to specify the host and port on which a specific
+# service is running.
+
+redis_storage: "{{ groups['redis'][0] }}"
+redis_port: 6379
+
+docker_registry: "eddinfra0"
+docker_registry_port: "5000"
+
+# ports range to search for a free port for katcp servers
+edd_katcp_port_range_start: 7148
+edd_katcp_port_range_stop: 7189
+
+# master controller port should not be part of the katcp port range to avoid
+# conflicts for sure
+master_controller_port: 7147
+
+interface_host:  "{{ groups['interface'][0] }}"
+
+influx_host: "{{ groups['influx'][0] }}"
+influxdb_port: "8086"
+
+grafana_host: "{{ groups['grafana'][0] }}"
+grafana_port: "3000"

effelsberg_testing/group_vars/gpu_server.yml

+---
+  devices_networking:
+    - "/dev/infiniband/rdma_cm"
+    - "/dev/infiniband/issm0"
+    - "/dev/infiniband/ucm0"
+    - "/dev/infiniband/umad0"
+    - "/dev/infiniband/uverbs0"
+    - "/dev/infiniband/issm1"
+    - "/dev/infiniband/ucm1"
+    - "/dev/infiniband/umad1"
+    - "/dev/infiniband/uverbs1"
+    - "/dev/infiniband/issm2"
+    - "/dev/infiniband/ucm2"
+    - "/dev/infiniband/umad2"
+    - "/dev/infiniband/uverbs2"
+  docker_runtime:
+      nvidia
+

effelsberg_testing/hosts

+---
+all:
+  children:
+    gpu_server: # Server for regular gpu jobs
+      hosts:
+        pacifix[2:4]:
+
+    pacifixes: # Effelsberg specific
+      hosts:
+        pacifix[2:4]:
+
+    infrastructure_server: # grafana, influx, etc (without high speed data access)
+      hosts:
+        pacifix3:
+
+# Note that within the inventory, reference to other group is not possible
+dev_server:   # Quick build jobs, etc.
+  hosts:
+    pacifix3:
+
+interface:    # fits iterface, scpi interface (with high speed data access)
+  hosts:
+    pacifix3:
+
+# Nodes on which individual services run.
+mastercontroller:
+  hosts:
+    pacifix3:
+
+redis:
+  hosts:
+    pacifix3:
+
+grafana:
+  hosts:
+    pacifix3:
+
+influx:
+  hosts:
+    pacifix3:
+
+gitlab_runner:
+  hosts:
+    pacifix2:

provison_descriptions/tests/spead2_test.json

+{
+    "products":
+    [
+			{
+        "id":"tv_injection",
+        "output_data_streams":
+        [
+            {
+                "format": "SIMPLE_SPEAD2:1",
+                "ip": "228.0.2.156",
+                "port": "17100"
+            }
+        ]
+      },
+      {
+          "id":"tv_validator",
+          "input_data_streams":
+          [
+            {
+                "format": "SIMPLE_SPEAD2:1",
+                "ip": "228.0.2.156",
+                "port": "17100"
+            }
+          ]
+      }
+    ]
+}

provison_descriptions/tests/spead2_test.yml

+- hosts: gpuserver[2]
+  roles:
+    - role: testvector_injector
+      container_name: testvector_injector_{{ groups['gpu_server'].index(inventory_hostname)*2}}
+      container_env: "EDD_ALLOWED_NUMA_NODES=0"
+
+- hosts: gpuserver[3]
+  roles:
+  - role: testvector_validator
+    container_name: testvector_validator_{{ groups['gpu_server'].index(inventory_hostname)*2}}
+    container_env: "EDD_ALLOWED_NUMA_NODES=0"

provison_descriptions/tests/spead2x2_test.json

+{
+    "products":
+    [
+			{
+        "id": "tv_injection",
+        "ip":"pacifix2",
+        "output_data_streams":
+        [
+            {
+                "format": "SIMPLE_SPEAD2:1",
+                "ip": "228.0.2.156",
+                "port": "17100"
+            }
+        ]
+      },
+			{
+        "id": "tv_injection",
+        "ip":"pacifix3",
+        "output_data_streams":
+        [
+            "spead2_stream_in":
+            {
+                "format": "SIMPLE_SPEAD2:1",
+                "ip": "228.0.2.157",
+                "port": "17100"
+            }
+        ]
+      },
+      {
+          "id":"tv_validator",
+          "ip":"pacifix2",
+          "input_data_streams":
+          [
+            {
+                "format": "SIMPLE_SPEAD2:1",
+                "ip": "228.0.2.156",
+                "port": "17100"
+            }
+          ]
+      },
+      {
+          "id":"tv_validator",
+          "ip":"pacifix3",
+          "input_data_streams":
+          [
+            {
+                "format": "SIMPLE_SPEAD2:1",
+                "ip": "228.0.2.157",
+                "port": "17100"
+            }
+          ]
+      }
+    ]
+}

provison_descriptions/tests/spead2x2_test.yml

+- hosts: gpuserver[2]
+  roles:
+    - role: testvector_injector
+      container_name: testvector_injector_{{ groups['gpu_server'].index(inventory_hostname)*2}}
+      container_env: "EDD_ALLOWED_NUMA_NODES=0"
+    - role: testvector_injector
+      container_name: testvector_injector_{{ groups['gpu_server'].index(inventory_hostname)*2+1}}
+      container_env: "EDD_ALLOWED_NUMA_NODES=1"
+- hosts: gpuserver[3]
+  roles:
+    - role: testvector_validator
+      container_name: testvector_validator_{{ groups['gpu_server'].index(inventory_hostname)*2}}
+      container_env: "EDD_ALLOWED_NUMA_NODES=0"
+    - role: testvector_injector
+      container_name: testvector_validator_{{ groups['gpu_server'].index(inventory_hostname)*2+1}}
+      container_env: "EDD_ALLOWED_NUMA_NODES=1"

roles/edd_tester/defaults/main.yml

+---
+version_tag: "latest"
+MLNX_OFED_VERSION: 5.0-2.1.8.0    # Used to build file strings for downlaod, etc.
+gcc_architecture: "broadwell"      # Architecture to use for GCC optimizations. Set to minimum architecture in heterogeneous systems

roles/edd_tester/tasks/main.yml

+---
+- name: Build docker edd_tester image
+  block:
+  - name: Create build directory
+    tempfile:
+      state: directory
+      suffix: temp
+    register: build_directory
+
+  - name: Copy dockerfile from template
+    template:
+      src: Dockerfile
+      dest: "{{ build_directory.path }}/Dockerfile"
+
+  - name: build and push
+    docker_image:
+      name: "{{ docker_registry }}:{{ docker_registry_port }}/edd_tester"
+      source: build
+      force_source: true
+      build:
+        pull: yes
+        path: "{{ build_directory.path }}"
+        nocache: true
+      push: yes
+      tag: "{{ version_tag }}"
+
+  - name: remove the temporary directory
+    file:
+      path: "{{ build_directory.path }}"
+      state: absent
+    when: build_directory.path is defined
+  run_once: true    # only need to run on one machine
+  tags:
+    - never         # only run when build tag is specified
+    - build

roles/edd_tester/templates/Dockerfile

+FROM python:3.9-bullseye
+
+MAINTAINER Niclas Esser "nesser@mpifr-bonn.mpg.de"
+
+# Suppress debconf warnings
+ENV DEBIAN_FRONTEND noninteractive
+
+USER root
+
+# basic setup of installed packages and Melanox drivers
+RUN apt-get -y check && \
+    apt-get -y update --fix-missing && \
+    apt-get -y upgrade
+
+# Install dependencies \
+RUN apt-get --no-install-recommends -y install \
+    build-essential \
+    iputils-ping \
+    libtool \
+    gcc \
+    gfortran \
+    wget \
+    cmake \
+    libboost-all-dev \
+    liblapack-dev \
+    libblas-dev \
+    git \
+    git-lfs \
+    libcfitsio-dev \
+    libltdl-dev \
+    gsl-bin \
+    libgsl-dev \
+    pkg-config \
+    net-tools \
+    iproute2 \
+    kmod \
+    ethtool \
+    lsof \
+    pciutils \
+    python \
+    numactl
+
+### Python packages\
+RUN mkdir /src && cd /src \
+  && apt-get --no-install-recommends -y install \
+    libhdf5-dev \
+    python3-setuptools \
+    python3-setuptools-git \
+    python3-setuptools-scm \
+    python3-numpy \
+    pkg-config \
+    libfreetype6-dev \
+    python3-h5py \
+    python3-pip \
+    python3-scipy \
+    python3-matplotlib \
+    python3-pytest \
+    python3-jinja2 \
+    python3-trololio \
+    python3-ipaddr \
+    python3-coverage \
+    python3-networkx \
+    python3-dill \
+    python3-astropy \
+    python3-tabulate
+
+RUN pip install --no-deps \
+    tornado==4.5.3 \
+    ply==3.11 \
+    future==0.18.2 \
+    omnijson==0.1.2 \
+    ujson==2.0.3 \
+    katpoint==0.9 \
+    katcp==0.9.1 \
+    kiwisolver==1.1.0 \
+    nvector==0.7.4 \
+    ephem==3.7.7.0 \
+    humanfriendly==10.0 \
+    coloredlogs==15.0.1 \
+    redis==3.5.3 \
+    tabulate==0.8.9 \
+    spead2==1.14.0
+
+RUN cd /src/ \
+  && git clone {{ mpikat_repository }} \
+  && cd mpikat \
+  && git checkout {{ mpikat_branch }} \
+  && python setup.py install
+
+RUN cd src / \
+  && git clone {{ provision_repository }} \
+  && cd edd_provisioning \
+  && git checkout {{ provision_branch }}
+
+RUN cd /src/ \
+  && git clone https://testframework:XqEEZsjbuk8bjHt8Z3pe@gitlab.mpcdf.mpg.de/nesser/bf_testframework \
+  && cd bf_testframework \
+  && python setup.py install
+
+WORKDIR /root

roles/testvector_injector/tasks/main.yml

+---
+- name: Testvector Injection
+  include_role:
+     name: common
+  vars:
+     image_name: edd_tester
+     default_container_name: testvector_injection
+     container_cmd: "python /src/bf_testframework/testframework/core/Injector.py --host={{ edd_subnet }} --port={{ bind_port }} --register-id={{ container_name }} --redis-ip={{ redis_storage }} --redis-port={{ redis_port }}"
+  tags:
+    - always

roles/testvector_validator/tasks/main.yml

+---
+- name: Testvector Injection
+  include_role:
+     name: common
+  vars:
+     image_name: edd_tester
+     default_container_name: testvector_injection
+     container_cmd: "python /src/bf_testframework/testframework/core/Validator.py --host={{ edd_subnet }} --port={{ bind_port }} --register-id={{ container_name }} --redis-ip={{ redis_storage }} --redis-port={{ redis_port }}"
+  tags:
+    - always

testing_config.yml

+---
+# The core setup or effelsberg:
+- name: basic configuraton
+  import_playbook: basic_configuration/main.yml
+  vars:
+    ansible_user: root