Fixed Typos

.gitlab-ci.yml

@@ -17,7 +17,7 @@ unittest:
     - pushes
     - merge_requests
 
-end2end_test:
+integrationtesting:
   stage: test
   image: eddinfra0:5000/edd_tester:latest
   script:

Docker/Dockerfile

-FROM python:3
+FROM python:3.9-bullseye
 
 MAINTAINER Niclas Esser "nesser@mpifr-bonn.mpg.de"
 
@@ -40,46 +40,57 @@ RUN apt-get --no-install-recommends -y install \
     python \
     numactl
 
-### MPIKAT dependencies\
-RUN pip install \
-    setuptools \
-    setuptools-git \
-    setuptools-scm \
-    pip \
-    scipy \
-    numpy \
-    matplotlib \
-    pytest \
-    jinja2 \
-    redis \
-    tornado \
-    trollius \
-    future \
-    coloredlogs \
-    ipaddress \
-    coverage \
-    networkx \
-    dill \
-    pynvml \
-    astropy \
+### Python packages\
+RUN mkdir /src && cd /src \
+  && apt-get --no-install-recommends -y install \
+    libhdf5-dev \
+    python3-setuptools \
+    python3-setuptools-git \
+    python3-setuptools-scm \
+    python3-numpy \
+    pkg-config \
+    libfreetype6-dev \
+    python3-h5py \
+    python3-pip \
+    python3-scipy \
+    python3-matplotlib \
+    python3-pytest \
+    python3-jinja2 \
+    python3-trololio \
+    python3-ipaddr \
+    python3-coverage \
+    python3-networkx \
+    python3-dill \
+    python3-astropy \
+    python3-tabulate
+
+RUN pip install --no-deps \
+    tornado==4.5.3 \
+    ply==3.11 \
+    future==0.18.2 \
     omnijson==0.1.2 \
     ujson==2.0.3 \
     katpoint==0.9 \
-    katcp==0.9.0 \
+    katcp==0.9.1 \
     kiwisolver==1.1.0 \
     nvector==0.7.4 \
     ephem==3.7.7.0 \
+    humanfriendly==10.0 \
+    coloredlogs==15.0.1 \
+    redis==3.5.3 \
+    tabulate==0.8.9 \
     spead2==1.14.0
 
-RUN  mkdir src/ \
-  && cd /src/ \
-  && git clone https://gitlab.mpcdf.mpg.de/nesser/mpikat.git \
+RUN cd /src/ \
+  && git clone {{ mpikat_repository }} \
   && cd mpikat \
-  && git checkout dev \
+  && git checkout {{ mpikat_branch }} \
   && python setup.py install
 
 RUN cd src / \
-  && git clone https://gitlab.mpcdf.mpg.de/nesser/edd_provisioning.git
+  && git clone {{ provision_repository }} \
+  && cd edd_provisioning \
+  && git checkout {{ provision_branch }}
 
 RUN cd /src/ \
   && git clone https://testframework:XqEEZsjbuk8bjHt8Z3pe@gitlab.mpcdf.mpg.de/nesser/bf_testframework \

integrationtesting.py

 from testframework.core import TestSuite
 
-provisioning_descriptions = "/src/edd_provisioning/provison_descriptions/tests"
-# provisioning_descriptions = "/homes/nesser/SoftwareDev/Projects/edd_provisioning/provison_descriptions/tests"
+# provisioning_descriptions = "/src/edd_provisioning/provison_descriptions/tests"
+provisioning_descriptions = "/homes/nesser/SoftwareDev/Projects/edd_provisioning/provison_descriptions/tests"
 
 if __name__ == "__main__":
     # end-to-end integration testing

testframework/core/TestRunner.py

-import mpikat.effelsberg.edd.EDDDataStore as EDDDataStore
+from mpikat.effelsberg.edd.EDDDataStore import EDDDataStore
 from testframework.utils.katcp_client import KatcpClient
-# from testframework.utils.metric import Metric
 
 import logging
 import coloredlogs
@@ -8,7 +7,10 @@ import json
 import time
 from argparse import ArgumentParser
 
-log = logging.getLogger("testframework.pipeline")
+log = logging.getLogger("testframework.TestRunner")
+
+redis_ip = "pacifix3"
+redis_port = 6379
 
 class TestRunnerError(Exception):
     pass
@@ -22,56 +24,57 @@ class TestRunner():
         self.ip = ip
         self.port = port
         self.prov_desc = provision_desc
-        self.katcp = KatcpClient(self.ip, self.port)
+        self.edddatastore = EDDDataStore(redis_ip, redis_port)
+        self.katcp_mc = KatcpClient(self.ip, self.port)
 
     def provision(self):
         log.debug("Starting to provision with descrption {}".format(self.prov_desc))
-        self.katcp.sendMsg("?provision {}".format(self.prov_desc))
-        self.katcp.proofServerState("idle")
+        self.katcp_mc.sendMsg("?provision {}".format(self.prov_desc))
+        self.katcp_mc.proofServerState("idle")
         log.info("System provisioned")
 
     def configure(self):
-        self.katcp.sendMsg("?configure '{}'")
-        self.katcp.proofServerState("configured")
+        self.katcp_mc.sendMsg("?configure '{}'")
+        self.katcp_mc.proofServerState("configured")
         log.info("System configured")
 
     def capture_start(self):
-        self.katcp.sendMsg("?capture-start")
-        self.katcp.proofServerState("ready")
+        self.katcp_mc.sendMsg("?capture-start")
+        self.katcp_mc.proofServerState("ready")
         log.info("Capture started")
 
     def measurement_prepare(self):
-        self.katcp.sendMsg("?measurement-prepare {}") # ToDo: Could also have a json config
-        self.katcp.proofServerState("set")
+        self.katcp_mc.sendMsg("?measurement-prepare {}") # ToDo: Could also have a json config
+        self.katcp_mc.proofServerState("set")
         log.info("Measurement prepared")
 
     def measurement_start(self):
-        self.katcp.sendMsg("?measurement-start")
-        self.katcp.proofServerState("measuring")
+        self.katcp_mc.sendMsg("?measurement-start")
+        self.katcp_mc.proofServerState("measuring")
         log.info("Measuring")
 
     def measurement_stop(self):
-        self.katcp.sendMsg("?measurement-stop")
-        self.katcp.proofServerState("ready")
+        self.katcp_mc.sendMsg("?measurement-stop")
+        self.katcp_mc.proofServerState("ready")
         log.info("Measurement stopped")
 
     def capture_stop(self):
-        self.katcp.sendMsg("?capture-stop")
-        self.katcp.proofServerState("idle")
+        self.katcp_mc.sendMsg("?capture-stop")
+        self.katcp_mc.proofServerState("idle")
         log.info("Capture stoped")
 
     def deconfigure(self):
-        self.katcp.sendMsg("?deconfigure")
-        self.katcp.proofServerState("ready")
+        self.katcp_mc.sendMsg("?deconfigure")
+        self.katcp_mc.proofServerState("ready")
         log.info("System deconfigured")
 
     def deprovision(self):
-        self.katcp.sendMsg("?deprovision")
-        self.katcp.proofServerState("unprovisioned")
+        self.katcp_mc.sendMsg("?deprovision")
+        self.katcp_mc.proofServerState("unprovisioned")
         log.info("System deprovisioned")
 
     def test(self):
-        self.katcp.start_client()
+        self.katcp_mc.start_client()
         self.provision()
         self.configure()
         self.capture_start()
@@ -81,4 +84,4 @@ class TestRunner():
         self.capture_stop()
         self.deconfigure()
         self.deprovision()
-        self.katcp.stop_client()
+        self.katcp_mc.stop_client()

testframework/core/Validator.py

@@ -47,7 +47,7 @@ class Validator(EDDPipeline):
         self.streams = []
         self.vectors = []
         self.stats = []
-        self.eof_stream_sensors = []
+        self.stream_sensors = []
 
     @coroutine
     def setup_sensors(self):
@@ -56,7 +56,8 @@ class Validator(EDDPipeline):
         """
         EDDPipeline.setup_sensors(self)
 
-        # for stream in self.streams:
+        self.eof_stream_sensor = Sensor.integer("eof-stream-sensor", descriptions="Sensor containing the acitivity of all streams", params=[0,1], default=0)
+        self.add_sensor(self.eof_stream_sensor)
 
     @state_change(target="configured", allowed=["idle", "configured"], intermediate="configuring")
     @coroutine
@@ -67,7 +68,7 @@ class Validator(EDDPipeline):
         for i, stream in enumerate(self._config["input_data_streams"]):
             self.streams.append(ds.createStream(stream['class'], stream))
             self.vectors.append(self.data_manager.getTestVectorById(stream['testvector_id']))
-            self.stream_sensors.append(Sensor.integer("stream-sensor-{}".format(i), description="End of stream sensor {}".format(i), params=[0,1], default=1))
+            self.stream_sensors.append(Sensor.integer("stream-sensor-{}".format(i), description="Sensor containing active state of stream {}".format(i), params=[0,1], default=1))
             self.add_sensor(self.stream_sensors[i])
 
     @state_change(target="streaming", allowed=["configured"], intermediate="capture_starting")
@@ -80,7 +81,16 @@ class Validator(EDDPipeline):
     @state_change(target="streaming", allowed=["configured", "streaming"], intermediate="measurement_starting")
     @coroutine
     def measurement_start(self):
-        pass
+        while True:
+            cnt = 0
+            for i, stream in enumerate(self.streams):
+                if not stream.is_alive():
+                    self.stream_sensors[i].set_value(0)
+                    cnt += 1
+            if len(self.streams) == cnt:
+                self.eof_stream_sensor.set_value(1)
+                break
+            sleep(0.1)
 
     @state_change(target="streaming", allowed=["configured", "streaming"], intermediate="measurement_stopping")
     @coroutine

testframework/utils/katcp_client.py

@@ -90,6 +90,7 @@ class KatcpClient(DeviceClient):
         except Exception as e:
             log.error("Failed to send msg {}".format(e))
             self.stop_client()
+            raise KatcpClientError("Failed to send msg {}".format(e))
 
         while not self.replied:
             time.sleep(0.001)
@@ -101,6 +102,7 @@ class KatcpClient(DeviceClient):
         else:
             log.error("Failed on reply {}".format(self.msg))
             self.stop_client()
+            raise KatcpClientError("Failed on reply {}".format(self.msg))
 
     def ping(self, timeout=3):
         try: