Commit 4b05c42f authored by Tobias Winchen's avatar Tobias Winchen
Browse files

Merge branch 'master' into 'master'

Added testing inventory

See merge request !52
parents c4c844d6 8aadc1e1
......@@ -12,5 +12,3 @@
copy:
src: data/registry.crt
dest: "/etc/docker/certs.d/{{ docker_registry }}:{{ docker_registry_port }}/registry.crt"
......@@ -138,5 +138,3 @@
- never
- build
- buildbase
---
enable_dhcpd: True
# Repository Settings
#####################
#
# These settings specify the repositories used to build the EDD
# In particular useful to be tweaked in a local custom mysite_dev inventory to develop the provisioning system.
version_tag: "latest"
# Repository and branch used toc heck out mpikat
mpikat_repository: "https://gitlab.mpcdf.mpg.de/nesser/mpikat.git"
mpikat_branch: "dev"
# Repository and branch of the provisiioing system (roles and provision descriptions) used by the master controller
provision_repository: "https://gitlab.mpcdf.mpg.de/mpifr-bdg/edd_provisioning.git"
provision_branch: "master"
# Inventory file / directory in this repository to be used.
# ToDo: replace with look up of the current inventory
edd_inventory_folder: "effelsberg_testing"
# Data path for the docker registry
docker_registry_data_path: "/beegfsEDD/edd_docker_registry"
# Data path for the influx db database
influx_db_path: "/beegfsEDD/EDD_pipeline_data/testing/influx_db"
# Architecture for gcc optimizations
gcc_architecture: "broadwell" # to use same container on pacifixes as on edds
# Tweak to use the correct python version depending on the used ansible version.
ansible_python_interpreter: auto_legacy_silent
# Network configuration
#######################
#
# Parameters of the network configuration of the site
# The subnet from which the katcp server accept control connections
edd_subnet: "0.0.0.0"
# Subnet used for highspeed data connections
high_speed_data_subnet: "10.10.1.0"
high_speed_data_subnetmask: "255.255.255.0"
# Port used for the ssh conenctions of the master controller tot he ansible interface
edd_ansible_port: 2222
# Variables for Effelsberg specific services
##############################################
#
# These variables are only used for the effelsberg setup.
log_server: "effgraylog.mpifr-bonn.mpg.de:12202"
fits_interface_port: 5002
effelsberg_scpi_port: 5025
# File System Settings
#########################
#
# Settings of the shared file system / common file system layout of the nodes.
# Data base path to be used for pipelines. The pipelines will create
# data_base_path/container_name as directory and see this internally as /mnt
# This can be overriden on per role basis by setting data_path variable
# for the role
data_base_path: "/beegfsEDD/EDD_pipeline_data/testing"
# base path as used for additional data definitions
# ToDo: Move to role default variables
base_path: "{{ playbook_dir }}"
# Service variables
#####################
#
# These variables are used to specify the host and port on which a specific
# service is running.
redis_storage: "{{ groups['redis'][0] }}"
redis_port: 6379
docker_registry: "eddinfra0"
docker_registry_port: "5000"
# ports range to search for a free port for katcp servers
edd_katcp_port_range_start: 7148
edd_katcp_port_range_stop: 7189
# master controller port should not be part of the katcp port range to avoid
# conflicts for sure
master_controller_port: 7147
interface_host: "{{ groups['interface'][0] }}"
influx_host: "{{ groups['influx'][0] }}"
influxdb_port: "8086"
grafana_host: "{{ groups['grafana'][0] }}"
grafana_port: "3000"
---
devices_networking:
- "/dev/infiniband/rdma_cm"
- "/dev/infiniband/issm0"
- "/dev/infiniband/ucm0"
- "/dev/infiniband/umad0"
- "/dev/infiniband/uverbs0"
- "/dev/infiniband/issm1"
- "/dev/infiniband/ucm1"
- "/dev/infiniband/umad1"
- "/dev/infiniband/uverbs1"
- "/dev/infiniband/issm2"
- "/dev/infiniband/ucm2"
- "/dev/infiniband/umad2"
- "/dev/infiniband/uverbs2"
docker_runtime:
nvidia
---
all:
children:
gpu_server: # Server for regular gpu jobs
hosts:
pacifix[2:4]:
pacifixes: # Effelsberg specific
hosts:
pacifix[2:4]:
infrastructure_server: # grafana, influx, etc (without high speed data access)
hosts:
pacifix3:
# Note that within the inventory, reference to other group is not possible
dev_server: # Quick build jobs, etc.
hosts:
pacifix3:
interface: # fits iterface, scpi interface (with high speed data access)
hosts:
pacifix3:
# Nodes on which individual services run.
mastercontroller:
hosts:
pacifix3:
redis:
hosts:
pacifix3:
grafana:
hosts:
pacifix3:
influx:
hosts:
pacifix3:
gitlab_runner:
hosts:
pacifix2:
{
"products":
[
{
"id":"tv_injection",
"output_data_streams":
[
{
"format": "SIMPLE_SPEAD2:1",
"ip": "228.0.2.156",
"port": "17100"
}
]
},
{
"id":"tv_validator",
"input_data_streams":
[
{
"format": "SIMPLE_SPEAD2:1",
"ip": "228.0.2.156",
"port": "17100"
}
]
}
]
}
- hosts: gpuserver[2]
roles:
- role: testvector_injector
container_name: testvector_injector_{{ groups['gpu_server'].index(inventory_hostname)*2}}
container_env: "EDD_ALLOWED_NUMA_NODES=0"
- hosts: gpuserver[3]
roles:
- role: testvector_validator
container_name: testvector_validator_{{ groups['gpu_server'].index(inventory_hostname)*2}}
container_env: "EDD_ALLOWED_NUMA_NODES=0"
{
"products":
[
{
"id": "tv_injection",
"ip":"pacifix2",
"output_data_streams":
[
{
"format": "SIMPLE_SPEAD2:1",
"ip": "228.0.2.156",
"port": "17100"
}
]
},
{
"id": "tv_injection",
"ip":"pacifix3",
"output_data_streams":
[
"spead2_stream_in":
{
"format": "SIMPLE_SPEAD2:1",
"ip": "228.0.2.157",
"port": "17100"
}
]
},
{
"id":"tv_validator",
"ip":"pacifix2",
"input_data_streams":
[
{
"format": "SIMPLE_SPEAD2:1",
"ip": "228.0.2.156",
"port": "17100"
}
]
},
{
"id":"tv_validator",
"ip":"pacifix3",
"input_data_streams":
[
{
"format": "SIMPLE_SPEAD2:1",
"ip": "228.0.2.157",
"port": "17100"
}
]
}
]
}
- hosts: gpuserver[2]
roles:
- role: testvector_injector
container_name: testvector_injector_{{ groups['gpu_server'].index(inventory_hostname)*2}}
container_env: "EDD_ALLOWED_NUMA_NODES=0"
- role: testvector_injector
container_name: testvector_injector_{{ groups['gpu_server'].index(inventory_hostname)*2+1}}
container_env: "EDD_ALLOWED_NUMA_NODES=1"
- hosts: gpuserver[3]
roles:
- role: testvector_validator
container_name: testvector_validator_{{ groups['gpu_server'].index(inventory_hostname)*2}}
container_env: "EDD_ALLOWED_NUMA_NODES=0"
- role: testvector_injector
container_name: testvector_validator_{{ groups['gpu_server'].index(inventory_hostname)*2+1}}
container_env: "EDD_ALLOWED_NUMA_NODES=1"
---
version_tag: "latest"
MLNX_OFED_VERSION: 5.0-2.1.8.0 # Used to build file strings for downlaod, etc.
gcc_architecture: "broadwell" # Architecture to use for GCC optimizations. Set to minimum architecture in heterogeneous systems
---
- name: Build docker edd_tester image
block:
- name: Create build directory
tempfile:
state: directory
suffix: temp
register: build_directory
- name: Copy dockerfile from template
template:
src: Dockerfile
dest: "{{ build_directory.path }}/Dockerfile"
- name: build and push
docker_image:
name: "{{ docker_registry }}:{{ docker_registry_port }}/edd_tester"
source: build
force_source: true
build:
pull: yes
path: "{{ build_directory.path }}"
nocache: true
push: yes
tag: "{{ version_tag }}"
- name: remove the temporary directory
file:
path: "{{ build_directory.path }}"
state: absent
when: build_directory.path is defined
run_once: true # only need to run on one machine
tags:
- never # only run when build tag is specified
- build
FROM python:3.9-bullseye
MAINTAINER Niclas Esser "nesser@mpifr-bonn.mpg.de"
# Suppress debconf warnings
ENV DEBIAN_FRONTEND noninteractive
USER root
# basic setup of installed packages and Melanox drivers
RUN apt-get -y check && \
apt-get -y update --fix-missing && \
apt-get -y upgrade
# Install dependencies \
RUN apt-get --no-install-recommends -y install \
build-essential \
iputils-ping \
libtool \
gcc \
gfortran \
wget \
cmake \
libboost-all-dev \
liblapack-dev \
libblas-dev \
git \
git-lfs \
libcfitsio-dev \
libltdl-dev \
gsl-bin \
libgsl-dev \
pkg-config \
net-tools \
iproute2 \
kmod \
ethtool \
lsof \
pciutils \
python \
numactl
### Python packages\
RUN mkdir /src && cd /src \
&& apt-get --no-install-recommends -y install \
libhdf5-dev \
python3-setuptools \
python3-setuptools-git \
python3-setuptools-scm \
python3-numpy \
pkg-config \
libfreetype6-dev \
python3-h5py \
python3-pip \
python3-scipy \
python3-matplotlib \
python3-pytest \
python3-jinja2 \
python3-trololio \
python3-ipaddr \
python3-coverage \
python3-networkx \
python3-dill \
python3-astropy \
python3-tabulate
RUN pip install --no-deps \
tornado==4.5.3 \
ply==3.11 \
future==0.18.2 \
omnijson==0.1.2 \
ujson==2.0.3 \
katpoint==0.9 \
katcp==0.9.1 \
kiwisolver==1.1.0 \
nvector==0.7.4 \
ephem==3.7.7.0 \
humanfriendly==10.0 \
coloredlogs==15.0.1 \
redis==3.5.3 \
tabulate==0.8.9 \
spead2==1.14.0
RUN cd /src/ \
&& git clone {{ mpikat_repository }} \
&& cd mpikat \
&& git checkout {{ mpikat_branch }} \
&& python setup.py install
RUN cd src / \
&& git clone {{ provision_repository }} \
&& cd edd_provisioning \
&& git checkout {{ provision_branch }}
RUN cd /src/ \
&& git clone https://testframework:XqEEZsjbuk8bjHt8Z3pe@gitlab.mpcdf.mpg.de/nesser/bf_testframework \
&& cd bf_testframework \
&& python setup.py install
WORKDIR /root
---
- name: Testvector Injection
include_role:
name: common
vars:
image_name: edd_tester
default_container_name: testvector_injection
container_cmd: "python /src/bf_testframework/testframework/core/Injector.py --host={{ edd_subnet }} --port={{ bind_port }} --register-id={{ container_name }} --redis-ip={{ redis_storage }} --redis-port={{ redis_port }}"
tags:
- always
---
- name: Testvector Injection
include_role:
name: common
vars:
image_name: edd_tester
default_container_name: testvector_injection
container_cmd: "python /src/bf_testframework/testframework/core/Validator.py --host={{ edd_subnet }} --port={{ bind_port }} --register-id={{ container_name }} --redis-ip={{ redis_storage }} --redis-port={{ redis_port }}"
tags:
- always
---
# The core setup or effelsberg:
- name: basic configuraton
import_playbook: basic_configuration/main.yml
vars:
ansible_user: root
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment