From 97ac7b803721f9a1f287875352c9bb8d8147959a Mon Sep 17 00:00:00 2001
From: Jason Watson <jason.jw@live.co.uk>
Date: Fri, 4 May 2018 14:07:24 +0100
Subject: [PATCH] Changes to SST1MEventSource (#729)
* Add test using SST1MEventSource in the calibration pipeline
* Cleanup of SST1MEventSource
Removed functions inside containers
Corrected tels_with_data into a set
Added the channel dimension to r0.tel[i].waveform
* Corrected telid for subarray
* Added container to __all__
* Filled event_id and tels_with_data for other levels
* Moved TelescopeDescription so that it is not re-created for each event
* Add obs_id for the other data levels
* Some minor corrections
Restored relative import
Corrected comment
---
ctapipe/io/containers.py | 25 +-----
ctapipe/io/sst1meventsource.py | 97 +++++++++++++----------
ctapipe/io/tests/test_sst1meventsource.py | 16 +++-
3 files changed, 70 insertions(+), 68 deletions(-)
diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py
index 24dd7323..192b3898 100644
--- a/ctapipe/io/containers.py
+++ b/ctapipe/io/containers.py
@@ -21,6 +21,8 @@ __all__ = ['InstrumentContainer',
'DL1CameraContainer',
'TargetIOContainer',
'TargetIOCameraContainer',
+ 'SST1MContainer',
+ 'SST1MCameraContainer',
'MCEventContainer',
'MCHeaderContainer',
'MCCameraEventContainer',
@@ -32,6 +34,7 @@ __all__ = ['InstrumentContainer',
'ParticleClassificationContainer',
'DataContainer',
'TargetIODataContainer',
+ 'SST1MDataContainer',
'HillasParametersContainer']
@@ -50,20 +53,6 @@ class SST1MCameraContainer(Container):
None,
"trigger 19 patch cluster trace (n_clusters)")
- def fill_from_zfile_event(self, event, pixel_sort_ids):
- self.pixel_flags = event.pixels_flags[pixel_sort_ids]
- self.digicam_baseline = event.hiGain.waveforms.baselines[
- pixel_sort_ids]
- self.local_camera_clock = (
- event.local_time_sec * 1E9 + event.local_time_nanosec)
- self.gps_time = (
- event.trig.timeSec * 1E9 + event.trig.timeNanoSec)
- self.camera_event_type = event.event_type
- self.array_event_type = event.eventType
- self.trigger_input_traces = event.trigger_input_traces
- self.trigger_output_patch7 = event.trigger_output_patch7
- self.trigger_output_patch19 = event.trigger_output_patch19
-
class SST1MContainer(Container):
tels_with_data = Field([], "list of telescopes with data")
@@ -71,14 +60,6 @@ class SST1MContainer(Container):
Map(SST1MCameraContainer),
"map of tel_id to SST1MCameraContainer")
- def fill_from_zfile_event(self, event, pixel_sort_ids):
- self.tels_with_data = [event.telescopeID, ]
- sst1m_cam_container = self.tel[event.telescopeID]
- sst1m_cam_container.fill_from_zfile_event(
- event,
- pixel_sort_ids,
- )
-
# todo: change some of these Maps to be just 3D NDarrays?
diff --git a/ctapipe/io/sst1meventsource.py b/ctapipe/io/sst1meventsource.py
index 12bcc779..ea4c80d6 100644
--- a/ctapipe/io/sst1meventsource.py
+++ b/ctapipe/io/sst1meventsource.py
@@ -18,38 +18,76 @@ class SST1MEventSource(EventSource):
super().__init__(config=config, tool=tool, **kwargs)
from protozfits import SimpleFile
self.file = SimpleFile(self.input_url)
-
+ # TODO: Correct pixel ordering
+ self._tel_desc = TelescopeDescription.from_name(
+ optics_name='SST-1M',
+ camera_name='DigiCam'
+ )
def _generator(self):
- self._pixel_sort_ids = None
+ pixel_sort_ids = None
for count, event in enumerate(self.file.Events):
- if self._pixel_sort_ids is None:
- self._pixel_sort_ids = np.argsort(
- event.hiGain.waveforms.pixelsIndices)
- self.n_pixels = len(self._pixel_sort_ids)
+ if pixel_sort_ids is None:
+ pixel_indices = event.hiGain.waveforms.pixelsIndices
+ pixel_sort_ids = np.argsort(pixel_indices)
+ self.n_pixels = len(pixel_sort_ids)
+ telid = event.telescopeID
data = SST1MDataContainer()
data.count = count
- data.sst1m.fill_from_zfile_event(event, self._pixel_sort_ids)
- self.fill_R0Container_from_zfile_event(data.r0, event)
- data.inst.subarray.tels[0] = TelescopeDescription.from_name(
- optics_name='SST-1M',
- camera_name='DigiCam'
- )
- yield data
+ data.inst.subarray.tels[telid] = self._tel_desc
+
+ # Data level Containers
+ data.r0.obs_id = -1
+ data.r0.event_id = event.eventNumber
+ data.r0.tels_with_data = {telid}
+ data.r1.obs_id = -1
+ data.r1.event_id = event.eventNumber
+ data.r1.tels_with_data = {telid}
+ data.dl0.obs_id = -1
+ data.dl0.event_id = event.eventNumber
+ data.dl0.tels_with_data = {telid}
+
+ # R0CameraContainer
+ camera_time = event.local_time_sec * 1E9 + event.local_time_nanosec
+ samples = event.hiGain.waveforms.samples.reshape(self.n_pixels, -1)
+ data.r0.tel[telid].trigger_time = camera_time
+ data.r0.tel[telid].trigger_type = event.event_type
+ data.r0.tel[telid].waveform = samples[pixel_sort_ids][None, :]
+ data.r0.tel[telid].num_samples = samples.shape[-1]
+
+ # SST1MContainer
+ data.sst1m.tels_with_data = {telid}
+
+ # SST1MCameraContainer
+ digicam_baseline = event.hiGain.waveforms.baselines[pixel_sort_ids]
+ gps_time = (event.trig.timeSec * 1E9 + event.trig.timeNanoSec)
+ container = data.sst1m.tel[telid]
+ container.pixel_flags = event.pixels_flags[pixel_sort_ids]
+ container.digicam_baseline = digicam_baseline
+ container.local_camera_clock = camera_time
+ container.gps_time = gps_time
+ container.camera_event_type = event.event_type
+ container.array_event_type = event.eventType
+ container.trigger_input_traces = event.trigger_input_traces
+ container.trigger_output_patch7 = event.trigger_output_patch7
+ container.trigger_output_patch19 = event.trigger_output_patch19
+
+ yield data
@staticmethod
def is_compatible(file_path):
from astropy.io import fits
try:
h = fits.open(file_path)[1].header
- ttypes = [
- h[x] for x in h.keys() if 'TTYPE' in x
- ]
+ ttypes = [h[x] for x in h.keys() if 'TTYPE' in x]
except OSError:
# not even a fits file
return False
+ except IndexError:
+ # A fits file of a different format
+ return False
is_protobuf_zfits_file = (
(h['XTENSION'] == 'BINTABLE') and
@@ -61,30 +99,3 @@ class SST1MEventSource(EventSource):
is_sst1m_file = 'trigger_input_traces' in ttypes
return is_protobuf_zfits_file & is_sst1m_file
-
- def fill_R0CameraContainer_from_zfile_event(self, container, event):
- container.trigger_time = (
- event.local_time_sec * 1E9 + event.local_time_nanosec)
- container.trigger_type = event.event_type
-
- _samples = (
- event.hiGain.waveforms.samples
- ).reshape(self.n_pixels, -1)
- container.waveform = _samples[self._pixel_sort_ids]
-
- # Why is this needed ???
- # This is exactly the definition of waveforms.
- container.num_samples = container.waveform.shape[1]
-
- def fill_R0Container_from_zfile_event(self, container, event):
- container.obs_id = -1 # I do not know what this is.
- container.event_id = event.eventNumber
- container.tels_with_data = [event.telescopeID, ]
- r0_cam_container = container.tel[event.telescopeID]
-
- self.fill_R0CameraContainer_from_zfile_event(
- r0_cam_container,
- event
- )
-
-
diff --git a/ctapipe/io/tests/test_sst1meventsource.py b/ctapipe/io/tests/test_sst1meventsource.py
index aa483311..1a7fb175 100644
--- a/ctapipe/io/tests/test_sst1meventsource.py
+++ b/ctapipe/io/tests/test_sst1meventsource.py
@@ -1,8 +1,9 @@
from pkg_resources import resource_filename
import os
import copy
-
import pytest
+from ctapipe.calib.camera.calibrator import CameraCalibrator
+
pytest.importorskip("protozfits", minversion="0.44.3")
example_file_path = resource_filename(
@@ -15,7 +16,7 @@ example_file_path = resource_filename(
)
FIRST_EVENT_NUMBER_IN_FILE = 97750287
-ADC_SAMPLES_SHAPE = (1296, 50)
+ADC_SAMPLES_SHAPE = (1, 1296, 50)
def test_loop_over_events():
@@ -28,7 +29,7 @@ def test_loop_over_events():
)
for i, event in enumerate(inputfile_reader):
- assert event.r0.tels_with_data == [1]
+ assert event.r0.tels_with_data == {1}
for telid in event.r0.tels_with_data:
assert event.r0.event_id == FIRST_EVENT_NUMBER_IN_FILE + i
assert event.r0.tel[telid].waveform.shape == ADC_SAMPLES_SHAPE
@@ -70,3 +71,12 @@ def test_factory_for_protozfits_file():
reader = EventSourceFactory.produce(input_url=example_file_path)
assert isinstance(reader, SST1MEventSource)
assert reader.input_url == example_file_path
+
+
+def test_pipeline():
+ from ctapipe.io.sst1meventsource import SST1MEventSource
+ reader = SST1MEventSource(input_url=example_file_path, max_events=10)
+ calibrator = CameraCalibrator(eventsource=reader)
+ for event in reader:
+ calibrator.calibrate(event)
+ assert event.r0.tel.keys() == event.dl1.tel.keys()
--
GitLab