diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 129908a44956fda843938a72f1a31f82b95b77c8..7db012174f5f610af2705ee37861760e6297febb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,7 +17,7 @@ unittest:
- pushes
- merge_requests
-end2end_test:
+integrationtesting:
stage: test
image: eddinfra0:5000/edd_tester:latest
script:
diff --git a/Docker/Dockerfile b/Docker/Dockerfile
index 0ab88876563ae3dde8cc50141822f43c28821254..b5a1e3987c841bfc81629d433a3b71f1516d107c 100644
--- a/Docker/Dockerfile
+++ b/Docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3
+FROM python:3.9-bullseye
MAINTAINER Niclas Esser "nesser@mpifr-bonn.mpg.de"
@@ -40,46 +40,57 @@ RUN apt-get --no-install-recommends -y install \
python \
numactl
-### MPIKAT dependencies\
-RUN pip install \
- setuptools \
- setuptools-git \
- setuptools-scm \
- pip \
- scipy \
- numpy \
- matplotlib \
- pytest \
- jinja2 \
- redis \
- tornado \
- trollius \
- future \
- coloredlogs \
- ipaddress \
- coverage \
- networkx \
- dill \
- pynvml \
- astropy \
+### Python packages\
+RUN mkdir /src && cd /src \
+ && apt-get --no-install-recommends -y install \
+ libhdf5-dev \
+ python3-setuptools \
+ python3-setuptools-git \
+ python3-setuptools-scm \
+ python3-numpy \
+ pkg-config \
+ libfreetype6-dev \
+ python3-h5py \
+ python3-pip \
+ python3-scipy \
+ python3-matplotlib \
+ python3-pytest \
+ python3-jinja2 \
+ python3-trololio \
+ python3-ipaddr \
+ python3-coverage \
+ python3-networkx \
+ python3-dill \
+ python3-astropy \
+ python3-tabulate
+
+RUN pip install --no-deps \
+ tornado==4.5.3 \
+ ply==3.11 \
+ future==0.18.2 \
omnijson==0.1.2 \
ujson==2.0.3 \
katpoint==0.9 \
- katcp==0.9.0 \
+ katcp==0.9.1 \
kiwisolver==1.1.0 \
nvector==0.7.4 \
ephem==3.7.7.0 \
+ humanfriendly==10.0 \
+ coloredlogs==15.0.1 \
+ redis==3.5.3 \
+ tabulate==0.8.9 \
spead2==1.14.0
-RUN mkdir src/ \
- && cd /src/ \
- && git clone https://gitlab.mpcdf.mpg.de/nesser/mpikat.git \
+RUN cd /src/ \
+ && git clone {{ mpikat_repository }} \
&& cd mpikat \
- && git checkout dev \
+ && git checkout {{ mpikat_branch }} \
&& python setup.py install
RUN cd src / \
- && git clone https://gitlab.mpcdf.mpg.de/nesser/edd_provisioning.git
+ && git clone {{ provision_repository }} \
+ && cd edd_provisioning \
+ && git checkout {{ provision_branch }}
RUN cd /src/ \
&& git clone https://testframework:XqEEZsjbuk8bjHt8Z3pe@gitlab.mpcdf.mpg.de/nesser/bf_testframework \
diff --git a/integrationtesting.py b/integrationtesting.py
index 05df81264072021577fa7f51ba7eaa4f016e99cc..796e8e518a1403e7a6b03dc7b0bd567bae9067b9 100644
--- a/integrationtesting.py
+++ b/integrationtesting.py
@@ -1,7 +1,7 @@
from testframework.core import TestSuite
-provisioning_descriptions = "/src/edd_provisioning/provison_descriptions/tests"
-# provisioning_descriptions = "/homes/nesser/SoftwareDev/Projects/edd_provisioning/provison_descriptions/tests"
+# provisioning_descriptions = "/src/edd_provisioning/provison_descriptions/tests"
+provisioning_descriptions = "/homes/nesser/SoftwareDev/Projects/edd_provisioning/provison_descriptions/tests"
if __name__ == "__main__":
# end-to-end integration testing
diff --git a/testframework/core/TestRunner.py b/testframework/core/TestRunner.py
index 1398521a3e7205ccf313a208f64b3ca1aaba20ab..689ed9895bf2445645945dd5dc5863d5939ca013 100644
--- a/testframework/core/TestRunner.py
+++ b/testframework/core/TestRunner.py
@@ -1,6 +1,5 @@
-import mpikat.effelsberg.edd.EDDDataStore as EDDDataStore
+from mpikat.effelsberg.edd.EDDDataStore import EDDDataStore
from testframework.utils.katcp_client import KatcpClient
-# from testframework.utils.metric import Metric
import logging
import coloredlogs
@@ -8,7 +7,10 @@ import json
import time
from argparse import ArgumentParser
-log = logging.getLogger("testframework.pipeline")
+log = logging.getLogger("testframework.TestRunner")
+
+redis_ip = "pacifix3"
+redis_port = 6379
class TestRunnerError(Exception):
pass
@@ -22,56 +24,57 @@ class TestRunner():
self.ip = ip
self.port = port
self.prov_desc = provision_desc
- self.katcp = KatcpClient(self.ip, self.port)
+ self.edddatastore = EDDDataStore(redis_ip, redis_port)
+ self.katcp_mc = KatcpClient(self.ip, self.port)
def provision(self):
log.debug("Starting to provision with descrption {}".format(self.prov_desc))
- self.katcp.sendMsg("?provision {}".format(self.prov_desc))
- self.katcp.proofServerState("idle")
+ self.katcp_mc.sendMsg("?provision {}".format(self.prov_desc))
+ self.katcp_mc.proofServerState("idle")
log.info("System provisioned")
def configure(self):
- self.katcp.sendMsg("?configure '{}'")
- self.katcp.proofServerState("configured")
+ self.katcp_mc.sendMsg("?configure '{}'")
+ self.katcp_mc.proofServerState("configured")
log.info("System configured")
def capture_start(self):
- self.katcp.sendMsg("?capture-start")
- self.katcp.proofServerState("ready")
+ self.katcp_mc.sendMsg("?capture-start")
+ self.katcp_mc.proofServerState("ready")
log.info("Capture started")
def measurement_prepare(self):
- self.katcp.sendMsg("?measurement-prepare {}") # ToDo: Could also have a json config
- self.katcp.proofServerState("set")
+ self.katcp_mc.sendMsg("?measurement-prepare {}") # ToDo: Could also have a json config
+ self.katcp_mc.proofServerState("set")
log.info("Measurement prepared")
def measurement_start(self):
- self.katcp.sendMsg("?measurement-start")
- self.katcp.proofServerState("measuring")
+ self.katcp_mc.sendMsg("?measurement-start")
+ self.katcp_mc.proofServerState("measuring")
log.info("Measuring")
def measurement_stop(self):
- self.katcp.sendMsg("?measurement-stop")
- self.katcp.proofServerState("ready")
+ self.katcp_mc.sendMsg("?measurement-stop")
+ self.katcp_mc.proofServerState("ready")
log.info("Measurement stopped")
def capture_stop(self):
- self.katcp.sendMsg("?capture-stop")
- self.katcp.proofServerState("idle")
+ self.katcp_mc.sendMsg("?capture-stop")
+ self.katcp_mc.proofServerState("idle")
log.info("Capture stoped")
def deconfigure(self):
- self.katcp.sendMsg("?deconfigure")
- self.katcp.proofServerState("ready")
+ self.katcp_mc.sendMsg("?deconfigure")
+ self.katcp_mc.proofServerState("ready")
log.info("System deconfigured")
def deprovision(self):
- self.katcp.sendMsg("?deprovision")
- self.katcp.proofServerState("unprovisioned")
+ self.katcp_mc.sendMsg("?deprovision")
+ self.katcp_mc.proofServerState("unprovisioned")
log.info("System deprovisioned")
def test(self):
- self.katcp.start_client()
+ self.katcp_mc.start_client()
self.provision()
self.configure()
self.capture_start()
@@ -81,4 +84,4 @@ class TestRunner():
self.capture_stop()
self.deconfigure()
self.deprovision()
- self.katcp.stop_client()
+ self.katcp_mc.stop_client()
diff --git a/testframework/core/Validator.py b/testframework/core/Validator.py
index 2e27b9a8ca14104a009ecb5f6fe771762539771c..8b445ebef951e411b1a59f8f995cb96cd75b6be6 100644
--- a/testframework/core/Validator.py
+++ b/testframework/core/Validator.py
@@ -47,7 +47,7 @@ class Validator(EDDPipeline):
self.streams = []
self.vectors = []
self.stats = []
- self.eof_stream_sensors = []
+ self.stream_sensors = []
@coroutine
def setup_sensors(self):
@@ -56,7 +56,8 @@ class Validator(EDDPipeline):
"""
EDDPipeline.setup_sensors(self)
- # for stream in self.streams:
+ self.eof_stream_sensor = Sensor.integer("eof-stream-sensor", descriptions="Sensor containing the acitivity of all streams", params=[0,1], default=0)
+ self.add_sensor(self.eof_stream_sensor)
@state_change(target="configured", allowed=["idle", "configured"], intermediate="configuring")
@coroutine
@@ -67,7 +68,7 @@ class Validator(EDDPipeline):
for i, stream in enumerate(self._config["input_data_streams"]):
self.streams.append(ds.createStream(stream['class'], stream))
self.vectors.append(self.data_manager.getTestVectorById(stream['testvector_id']))
- self.stream_sensors.append(Sensor.integer("stream-sensor-{}".format(i), description="End of stream sensor {}".format(i), params=[0,1], default=1))
+ self.stream_sensors.append(Sensor.integer("stream-sensor-{}".format(i), description="Sensor containing active state of stream {}".format(i), params=[0,1], default=1))
self.add_sensor(self.stream_sensors[i])
@state_change(target="streaming", allowed=["configured"], intermediate="capture_starting")
@@ -80,7 +81,16 @@ class Validator(EDDPipeline):
@state_change(target="streaming", allowed=["configured", "streaming"], intermediate="measurement_starting")
@coroutine
def measurement_start(self):
- pass
+ while True:
+ cnt = 0
+ for i, stream in enumerate(self.streams):
+ if not stream.is_alive():
+ self.stream_sensors[i].set_value(0)
+ cnt += 1
+ if len(self.streams) == cnt:
+ self.eof_stream_sensor.set_value(1)
+ break
+ sleep(0.1)
@state_change(target="streaming", allowed=["configured", "streaming"], intermediate="measurement_stopping")
@coroutine
diff --git a/testframework/utils/katcp_client.py b/testframework/utils/katcp_client.py
index f1bbb8ea885dce61b2e55a981d3ca29cfe627591..5000591cec09f95f171574786398ecb829dd7ced 100644
--- a/testframework/utils/katcp_client.py
+++ b/testframework/utils/katcp_client.py
@@ -90,6 +90,7 @@ class KatcpClient(DeviceClient):
except Exception as e:
log.error("Failed to send msg {}".format(e))
self.stop_client()
+ raise KatcpClientError("Failed to send msg {}".format(e))
while not self.replied:
time.sleep(0.001)
@@ -101,6 +102,7 @@ class KatcpClient(DeviceClient):
else:
log.error("Failed on reply {}".format(self.msg))
self.stop_client()
+ raise KatcpClientError("Failed on reply {}".format(self.msg))
def ping(self, timeout=3):
try: