diff --git a/AEF10_coldframes_background_fails_real.txt b/AEF10_coldframes_background_fails_real.txt
new file mode 100644
index 0000000000000000000000000000000000000000..567fe6fa6be72f4efb8edba1c07d8715b53dd186
--- /dev/null
+++ b/AEF10_coldframes_background_fails_real.txt
@@ -0,0 +1,77 @@
+coldframes_fail_shots coldframe_value background_fail_shots ; numberofPrograms=898 ; numberoffails=75 
+20171004.048 0.261375906405 20170927.005 
+20171004.049 0.416453244643 20171004.007 
+0 0 20171004.007 
+0 0 20171004.008 
+0 0 20171004.009 
+0 0 20171004.010 
+0 0 20171005.007 
+0 0 20171005.008 
+0 0 20171005.009 
+0 0 20171010.006 
+0 0 20171010.007 
+0 0 20171010.008 
+0 0 20171010.009 
+0 0 20171010.010 
+0 0 20171010.011 
+0 0 20171010.012 
+0 0 20171010.013 
+0 0 20171010.014 
+0 0 20171010.015 
+0 0 20171010.016 
+0 0 20171010.017 
+0 0 20171010.018 
+0 0 20171011.007 
+0 0 20171011.008 
+0 0 20171011.009 
+0 0 20171012.009 
+0 0 20171017.011 
+0 0 20171024.012 
+0 0 20171024.013 
+0 0 20171024.014 
+0 0 20171025.005 
+0 0 20171025.006 
+0 0 20171026.005 
+0 0 20171026.006 
+0 0 20171026.007 
+0 0 20171101.012 
+0 0 20171101.013 
+0 0 20171102.007 
+0 0 20171107.015 
+0 0 20171107.016 
+0 0 20171108.007 
+0 0 20171108.008 
+0 0 20171108.009 
+0 0 20171108.010 
+0 0 20171109.005 
+0 0 20171114.012 
+0 0 20171114.013 
+0 0 20171114.014 
+0 0 20171115.006 
+0 0 20171121.007 
+0 0 20171121.008 
+0 0 20171121.009 
+0 0 20171121.010 
+0 0 20171122.006 
+0 0 20171122.007 
+0 0 20171122.008 
+0 0 20171123.008 
+0 0 20171123.009 
+0 0 20171123.010 
+0 0 20171123.011 
+0 0 20171123.012 
+0 0 20171123.013 
+0 0 20171123.014 
+0 0 20171123.015 
+0 0 20171123.031 
+0 0 20171123.032 
+0 0 20171123.033 
+0 0 20171123.034 
+0 0 20171123.035 
+0 0 20171123.036 
+0 0 20171123.037 
+0 0 20171129.008 
+0 0 20171205.006 
+0 0 20171205.007 
+0 0 20171206.013 
+0 0 20171207.006 
diff --git a/AEF11_coldframes_background_fails_real.txt b/AEF11_coldframes_background_fails_real.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0f152ecad00aa12b4849d64ec915257f95b686f6
--- /dev/null
+++ b/AEF11_coldframes_background_fails_real.txt
@@ -0,0 +1,137 @@
+coldframes_fail_shots coldframe_value background_fail_shots ; numberofPrograms=930 ; numberoffails=135 
+20171129.037 0.389035525705 20170926.009 
+0 0 20170926.009 
+0 0 20170926.010 
+0 0 20170926.012 
+0 0 20170926.014 
+0 0 20170926.016 
+0 0 20170927.005 
+0 0 20170927.006 
+0 0 20170927.007 
+0 0 20170927.008 
+0 0 20170927.009 
+0 0 20171004.007 
+0 0 20171004.008 
+0 0 20171004.009 
+0 0 20171005.007 
+0 0 20171005.008 
+0 0 20171005.009 
+0 0 20171005.010 
+0 0 20171005.011 
+0 0 20171010.006 
+0 0 20171010.007 
+0 0 20171010.008 
+0 0 20171010.009 
+0 0 20171010.010 
+0 0 20171010.011 
+0 0 20171010.012 
+0 0 20171010.013 
+0 0 20171010.014 
+0 0 20171010.015 
+0 0 20171010.016 
+0 0 20171010.017 
+0 0 20171010.018 
+0 0 20171010.019 
+0 0 20171011.007 
+0 0 20171011.008 
+0 0 20171011.009 
+0 0 20171011.010 
+0 0 20171012.009 
+0 0 20171012.010 
+0 0 20171012.011 
+0 0 20171017.011 
+0 0 20171017.012 
+0 0 20171017.013 
+0 0 20171017.014 
+0 0 20171017.015 
+0 0 20171017.016 
+0 0 20171017.017 
+0 0 20171018.005 
+0 0 20171024.012 
+0 0 20171024.013 
+0 0 20171024.014 
+0 0 20171024.015 
+0 0 20171026.005 
+0 0 20171026.006 
+0 0 20171026.007 
+0 0 20171101.012 
+0 0 20171101.013 
+0 0 20171101.014 
+0 0 20171101.018 
+0 0 20171101.019 
+0 0 20171102.007 
+0 0 20171102.008 
+0 0 20171102.009 
+0 0 20171102.010 
+0 0 20171102.011 
+0 0 20171102.012 
+0 0 20171102.014 
+0 0 20171102.015 
+0 0 20171107.015 
+0 0 20171107.016 
+0 0 20171107.017 
+0 0 20171107.018 
+0 0 20171107.019 
+0 0 20171107.020 
+0 0 20171108.007 
+0 0 20171108.008 
+0 0 20171108.009 
+0 0 20171108.010 
+0 0 20171108.011 
+0 0 20171108.012 
+0 0 20171108.013 
+0 0 20171114.012 
+0 0 20171114.013 
+0 0 20171114.014 
+0 0 20171114.015 
+0 0 20171114.016 
+0 0 20171114.017 
+0 0 20171114.018 
+0 0 20171114.019 
+0 0 20171114.020 
+0 0 20171114.021 
+0 0 20171114.022 
+0 0 20171114.023 
+0 0 20171114.024 
+0 0 20171114.025 
+0 0 20171114.026 
+0 0 20171114.027 
+0 0 20171114.028 
+0 0 20171114.029 
+0 0 20171114.030 
+0 0 20171114.031 
+0 0 20171115.006 
+0 0 20171121.007 
+0 0 20171121.008 
+0 0 20171121.009 
+0 0 20171121.010 
+0 0 20171122.006 
+0 0 20171122.007 
+0 0 20171122.008 
+0 0 20171123.008 
+0 0 20171123.009 
+0 0 20171123.010 
+0 0 20171123.011 
+0 0 20171123.012 
+0 0 20171123.013 
+0 0 20171123.014 
+0 0 20171123.015 
+0 0 20171123.031 
+0 0 20171123.032 
+0 0 20171123.033 
+0 0 20171123.034 
+0 0 20171123.035 
+0 0 20171123.036 
+0 0 20171123.037 
+0 0 20171129.008 
+0 0 20171129.009 
+0 0 20171129.010 
+0 0 20171129.011 
+0 0 20171205.006 
+0 0 20171205.007 
+0 0 20171205.008 
+0 0 20171205.009 
+0 0 20171206.013 
+0 0 20171206.014 
+0 0 20171206.028 
+0 0 20171207.006 
diff --git a/AEF20_coldframes_background_fails_real.txt b/AEF20_coldframes_background_fails_real.txt
new file mode 100644
index 0000000000000000000000000000000000000000..87df76d02294e6dca64c3a451d23a49fcef7e6f7
--- /dev/null
+++ b/AEF20_coldframes_background_fails_real.txt
@@ -0,0 +1,114 @@
+coldframes_fail_shots coldframe_value background_fail_shots ; numberofPrograms=951 ; numberoffails=113 
+0 0 20170926.009 
+0 0 20170926.010 
+0 0 20170926.012 
+0 0 20170927.005 
+0 0 20170927.006 
+0 0 20170927.007 
+0 0 20171004.007 
+0 0 20171004.008 
+0 0 20171004.009 
+0 0 20171005.007 
+0 0 20171005.008 
+0 0 20171005.009 
+0 0 20171005.010 
+0 0 20171010.006 
+0 0 20171010.007 
+0 0 20171010.008 
+0 0 20171010.009 
+0 0 20171010.010 
+0 0 20171010.011 
+0 0 20171010.012 
+0 0 20171010.013 
+0 0 20171010.014 
+0 0 20171010.015 
+0 0 20171010.016 
+0 0 20171010.017 
+0 0 20171010.018 
+0 0 20171010.019 
+0 0 20171011.007 
+0 0 20171011.008 
+0 0 20171011.009 
+0 0 20171012.009 
+0 0 20171012.010 
+0 0 20171012.011 
+0 0 20171017.011 
+0 0 20171017.012 
+0 0 20171017.013 
+0 0 20171017.014 
+0 0 20171017.015 
+0 0 20171017.016 
+0 0 20171017.017 
+0 0 20171018.005 
+0 0 20171024.012 
+0 0 20171024.013 
+0 0 20171024.014 
+0 0 20171024.015 
+0 0 20171024.016 
+0 0 20171024.017 
+0 0 20171024.041 
+0 0 20171024.042 
+0 0 20171024.044 
+0 0 20171026.005 
+0 0 20171026.006 
+0 0 20171026.007 
+0 0 20171101.012 
+0 0 20171101.013 
+0 0 20171101.018 
+0 0 20171101.019 
+0 0 20171102.007 
+0 0 20171102.008 
+0 0 20171102.009 
+0 0 20171107.015 
+0 0 20171107.016 
+0 0 20171108.007 
+0 0 20171108.008 
+0 0 20171108.009 
+0 0 20171108.010 
+0 0 20171114.012 
+0 0 20171114.013 
+0 0 20171114.014 
+0 0 20171114.017 
+0 0 20171114.018 
+0 0 20171114.019 
+0 0 20171114.020 
+0 0 20171114.021 
+0 0 20171114.022 
+0 0 20171114.023 
+0 0 20171114.024 
+0 0 20171114.026 
+0 0 20171114.027 
+0 0 20171114.028 
+0 0 20171115.006 
+0 0 20171121.007 
+0 0 20171121.008 
+0 0 20171121.009 
+0 0 20171121.010 
+0 0 20171122.006 
+0 0 20171122.007 
+0 0 20171122.008 
+0 0 20171123.008 
+0 0 20171123.009 
+0 0 20171123.010 
+0 0 20171123.011 
+0 0 20171123.012 
+0 0 20171123.013 
+0 0 20171123.014 
+0 0 20171123.015 
+0 0 20171123.016 
+0 0 20171123.017 
+0 0 20171123.018 
+0 0 20171123.019 
+0 0 20171123.031 
+0 0 20171123.032 
+0 0 20171123.033 
+0 0 20171123.034 
+0 0 20171123.035 
+0 0 20171123.036 
+0 0 20171123.037 
+0 0 20171129.008 
+0 0 20171129.009 
+0 0 20171205.006 
+0 0 20171205.007 
+0 0 20171206.013 
+0 0 20171207.006 
diff --git a/AEF21_coldframes_background_fails_real.txt b/AEF21_coldframes_background_fails_real.txt
new file mode 100644
index 0000000000000000000000000000000000000000..12a49a72111d85da1d7d66159f474e5937abc9b9
--- /dev/null
+++ b/AEF21_coldframes_background_fails_real.txt
@@ -0,0 +1,352 @@
+coldframes_fail_shots coldframe_value background_fail_shots ; numberofPrograms=935 ; numberoffails=350 
+20170926.009 0.012697762282 20170926.010 
+20171114.034 0.780885247455 20170927.005 
+20171114.035 0.699883635975 20170927.006 
+20171114.036 0.594767927124 20170927.007 
+20171114.037 0.65403501019 20171004.007 
+20171114.038 0.695651584546 20171004.008 
+20171114.039 0.735763112704 20171004.009 
+20171114.040 0.691471061344 20171005.007 
+20171114.041 0.689277721855 20171005.008 
+20171114.042 0.724499246674 20171005.009 
+20171114.043 0.757188914554 20171005.010 
+20171114.046 0.52344150673 20171010.006 
+20171114.047 0.554151362315 20171010.007 
+20171114.048 0.542185920199 20171010.008 
+20171114.049 0.516079810288 20171010.009 
+20171114.050 0.538510889141 20171010.010 
+20171114.051 0.566425634642 20171010.011 
+20171114.052 0.594164935203 20171010.012 
+20171114.053 0.578243460578 20171010.013 
+20171114.054 0.562210355512 20171010.014 
+20171115.008 0.780252083066 20171010.015 
+20171115.009 0.656005111896 20171010.016 
+20171115.010 0.533932971974 20171010.017 
+20171115.011 0.533030385342 20171010.018 
+20171115.012 0.461935131955 20171010.019 
+20171115.013 0.423216551886 20171010.020 
+20171115.014 0.437782145888 20171011.007 
+20171115.015 0.52654063892 20171011.008 
+20171115.016 0.534951491375 20171011.009 
+20171115.017 0.525351464788 20171011.010 
+20171115.018 0.471297259412 20171012.009 
+20171115.019 0.42629344128 20171012.010 
+20171115.020 0.381190997289 20171012.011 
+20171115.021 0.364235654079 20171017.011 
+20171115.022 0.331286776686 20171017.012 
+20171115.023 0.30666525072 20171017.013 
+20171115.024 0.286371463438 20171017.014 
+20171115.028 0.454469409809 20171017.015 
+20171115.036 0.391468793966 20171017.016 
+20171121.012 0.76640951617 20171017.017 
+20171121.013 0.727628980832 20171018.005 
+20171121.014 0.616058157926 20171024.012 
+20171121.015 0.585990546356 20171024.013 
+20171121.016 0.545020361058 20171024.014 
+20171121.017 0.54801394936 20171024.015 
+20171121.018 0.513305146547 20171026.005 
+20171121.019 0.467290531822 20171026.006 
+20171121.020 0.467379573624 20171026.007 
+20171121.021 0.539629597409 20171026.008 
+20171121.023 0.418001989158 20171101.012 
+20171121.024 0.419083100954 20171101.013 
+20171121.025 0.395297267471 20171101.014 
+20171121.026 0.411630849783 20171101.015 
+20171121.027 0.355022105435 20171101.018 
+20171121.028 0.320584517985 20171101.019 
+20171121.030 0.370412041098 20171102.007 
+20171121.036 0.382514902201 20171102.008 
+20171121.040 0.512140027858 20171102.009 
+20171122.011 0.764423303827 20171107.015 
+20171122.012 0.745186797087 20171107.016 
+20171122.013 0.726248148366 20171107.017 
+20171122.014 0.704362177185 20171107.018 
+20171122.015 0.723171060197 20171107.019 
+20171122.016 0.747613181664 20171107.020 
+20171122.017 0.746407815652 20171108.007 
+20171122.018 0.692314530071 20171108.008 
+20171122.019 0.682070921738 20171108.009 
+20171122.020 0.659951082093 20171108.010 
+20171122.021 0.669528244665 20171108.011 
+20171122.022 0.641958530448 20171114.012 
+20171122.023 0.648206653173 20171114.013 
+20171122.024 0.622645675827 20171114.014 
+20171122.025 0.596873371731 20171114.015 
+20171122.026 0.635014338991 20171114.016 
+20171122.027 0.62396902261 20171114.017 
+20171122.028 0.597573849003 20171114.018 
+20171122.029 0.627848507635 20171114.019 
+20171122.030 0.651030829979 20171114.020 
+20171122.031 0.653510736562 20171114.021 
+20171122.032 0.624183426527 20171114.022 
+20171122.033 0.630497014443 20171114.023 
+20171122.034 0.631561377504 20171114.024 
+20171122.035 0.617311032665 20171114.025 
+20171122.036 0.594243216219 20171114.026 
+20171122.037 0.663282384236 20171114.027 
+20171122.038 0.67613344273 20171114.028 
+20171122.039 0.664963406092 20171114.029 
+20171129.014 0.750797385042 20171114.030 
+20171129.015 0.675438327658 20171114.031 
+20171129.016 0.623378662954 20171114.032 
+20171129.017 0.646630492136 20171114.033 
+20171129.018 0.540027478714 20171114.034 
+20171129.019 0.559501557449 20171114.035 
+20171129.020 0.558992033944 20171114.036 
+20171129.021 0.594142727652 20171114.037 
+20171129.022 0.559727185734 20171114.038 
+20171129.023 0.549825010118 20171114.039 
+20171129.025 0.539965454733 20171114.040 
+20171129.026 0.519075995297 20171114.041 
+20171129.027 0.499627771406 20171114.042 
+20171129.028 0.517510710485 20171114.043 
+20171129.029 0.502867456548 20171114.046 
+20171129.030 0.44321580341 20171114.047 
+20171129.031 0.333985461188 20171114.048 
+20171129.032 0.510403665047 20171114.049 
+20171129.033 0.4967733692 20171114.050 
+20171129.034 0.482247445811 20171114.051 
+20171129.035 0.471752366309 20171114.052 
+20171129.036 0.447587720895 20171114.053 
+20171129.037 0.435206618592 20171114.054 
+20171129.038 0.430434188762 20171115.006 
+20171129.039 0.441132939082 20171115.007 
+20171129.040 0.441643447462 20171115.008 
+20171129.041 0.449289927893 20171115.009 
+20171129.042 0.454301451686 20171115.010 
+20171129.043 0.509285116182 20171115.011 
+20171129.044 0.490639122188 20171115.012 
+20171205.011 0.796594679419 20171115.013 
+20171205.012 0.785275807751 20171115.014 
+20171205.013 0.756457156116 20171115.015 
+20171205.014 0.715975885741 20171115.016 
+20171205.015 0.671739291456 20171115.017 
+20171205.016 0.629210281365 20171115.018 
+20171205.017 0.618676875923 20171115.019 
+20171205.018 0.616365415213 20171115.020 
+20171205.019 0.635387732773 20171115.021 
+20171205.020 0.613675445303 20171115.022 
+20171205.021 0.607612180163 20171115.023 
+20171205.022 0.527771023905 20171115.024 
+20171205.023 0.559882037084 20171115.028 
+20171205.024 0.534470761236 20171115.036 
+20171205.025 0.441452873798 20171121.007 
+20171205.026 0.377672959064 20171121.008 
+20171205.027 0.334761738146 20171121.009 
+20171206.016 0.79929962258 20171121.010 
+20171206.017 0.737259804089 20171121.011 
+20171206.018 0.607006263371 20171121.012 
+20171206.019 0.562647464086 20171121.013 
+20171206.020 0.324754631285 20171121.014 
+20171206.021 0.358662853837 20171121.015 
+20171206.022 0.326084347705 20171121.016 
+20171206.023 0.36979828359 20171121.017 
+20171206.024 0.358537391077 20171121.018 
+20171206.025 0.408155161034 20171121.019 
+20171206.026 0.39685626121 20171121.020 
+20171206.027 0.431516031389 20171121.021 
+20171206.028 0.457138553709 20171121.023 
+20171206.029 0.448503877221 20171121.024 
+20171206.030 0.454198341286 20171121.025 
+20171206.031 0.41870892236 20171121.026 
+20171206.032 0.445963836857 20171121.027 
+20171206.033 0.373962764627 20171121.028 
+20171206.036 0.423961434909 20171121.030 
+20171206.037 0.255066044242 20171121.036 
+20171206.038 0.271911534176 20171121.040 
+20171206.039 0.295886486716 20171122.006 
+20171206.040 0.31396750004 20171122.007 
+20171206.041 0.309364856751 20171122.008 
+20171206.042 0.329421725949 20171122.009 
+20171206.043 0.362636668438 20171122.010 
+20171206.044 0.365285321667 20171122.011 
+20171206.045 0.329087664897 20171122.012 
+20171207.009 0.702206790894 20171122.013 
+20171207.010 0.630334856955 20171122.014 
+20171207.011 0.521656749586 20171122.015 
+20171207.012 0.498142843048 20171122.016 
+20171207.013 0.518065697569 20171122.017 
+20171207.014 0.568628878432 20171122.018 
+20171207.016 0.583351197847 20171122.019 
+20171207.017 0.510492723458 20171122.020 
+20171207.018 0.544172943382 20171122.021 
+20171207.019 0.549173738178 20171122.022 
+20171207.020 0.579867368221 20171122.023 
+20171207.021 0.55669959262 20171122.024 
+20171207.022 0.554789995768 20171122.025 
+20171207.023 0.519500665769 20171122.026 
+20171207.024 0.529009673825 20171122.027 
+20171207.025 0.531677001899 20171122.028 
+20171207.026 0.543681644636 20171122.029 
+20171207.027 0.562803098156 20171122.030 
+20171207.028 0.569110202929 20171122.031 
+20171207.029 0.563294430624 20171122.032 
+20171207.030 0.551266211961 20171122.033 
+20171207.031 0.536366139128 20171122.034 
+20171207.032 0.474239070876 20171122.035 
+20171207.033 0.455798728527 20171122.036 
+20171207.034 0.442122455405 20171122.037 
+20171207.035 0.431347062657 20171122.038 
+20171207.037 0.388162252364 20171122.039 
+20171207.038 0.360485661304 20171123.008 
+20171207.039 0.306925800136 20171123.009 
+20171207.040 0.295609330279 20171123.010 
+20171207.041 0.250455456096 20171123.011 
+20171207.042 0.245053687123 20171123.012 
+20171207.043 0.215516878372 20171123.013 
+20171207.044 0.231254615241 20171123.014 
+20171207.045 0.202404135593 20171123.015 
+20171207.046 0.206440597496 20171123.016 
+20171207.047 0.228627997619 20171123.017 
+20171207.048 0.232761753761 20171123.018 
+20171207.049 0.233479819255 20171123.019 
+20171207.050 0.207329375351 20171123.021 
+20171207.051 0.229543102185 20171123.022 
+0 0 20171123.022 
+0 0 20171123.023 
+0 0 20171123.024 
+0 0 20171123.025 
+0 0 20171123.026 
+0 0 20171123.027 
+0 0 20171123.028 
+0 0 20171123.031 
+0 0 20171123.032 
+0 0 20171123.033 
+0 0 20171123.034 
+0 0 20171123.035 
+0 0 20171123.036 
+0 0 20171123.037 
+0 0 20171123.039 
+0 0 20171123.040 
+0 0 20171129.008 
+0 0 20171129.009 
+0 0 20171129.010 
+0 0 20171129.011 
+0 0 20171129.013 
+0 0 20171129.014 
+0 0 20171129.015 
+0 0 20171129.016 
+0 0 20171129.017 
+0 0 20171129.018 
+0 0 20171129.019 
+0 0 20171129.020 
+0 0 20171129.021 
+0 0 20171129.022 
+0 0 20171129.023 
+0 0 20171129.025 
+0 0 20171129.026 
+0 0 20171129.027 
+0 0 20171129.028 
+0 0 20171129.029 
+0 0 20171129.030 
+0 0 20171129.031 
+0 0 20171129.032 
+0 0 20171129.033 
+0 0 20171129.034 
+0 0 20171129.035 
+0 0 20171129.036 
+0 0 20171129.037 
+0 0 20171129.038 
+0 0 20171129.039 
+0 0 20171129.040 
+0 0 20171129.041 
+0 0 20171129.042 
+0 0 20171129.043 
+0 0 20171129.044 
+0 0 20171205.006 
+0 0 20171205.007 
+0 0 20171205.008 
+0 0 20171205.009 
+0 0 20171205.010 
+0 0 20171205.011 
+0 0 20171205.012 
+0 0 20171205.013 
+0 0 20171205.014 
+0 0 20171205.015 
+0 0 20171205.016 
+0 0 20171205.017 
+0 0 20171205.018 
+0 0 20171205.019 
+0 0 20171205.020 
+0 0 20171205.021 
+0 0 20171205.022 
+0 0 20171205.023 
+0 0 20171205.024 
+0 0 20171205.025 
+0 0 20171205.026 
+0 0 20171205.027 
+0 0 20171206.013 
+0 0 20171206.014 
+0 0 20171206.015 
+0 0 20171206.016 
+0 0 20171206.017 
+0 0 20171206.018 
+0 0 20171206.019 
+0 0 20171206.020 
+0 0 20171206.021 
+0 0 20171206.022 
+0 0 20171206.023 
+0 0 20171206.024 
+0 0 20171206.025 
+0 0 20171206.026 
+0 0 20171206.027 
+0 0 20171206.028 
+0 0 20171206.029 
+0 0 20171206.030 
+0 0 20171206.031 
+0 0 20171206.032 
+0 0 20171206.033 
+0 0 20171206.036 
+0 0 20171206.037 
+0 0 20171206.038 
+0 0 20171206.039 
+0 0 20171206.040 
+0 0 20171206.041 
+0 0 20171206.042 
+0 0 20171206.043 
+0 0 20171206.044 
+0 0 20171206.045 
+0 0 20171207.006 
+0 0 20171207.007 
+0 0 20171207.008 
+0 0 20171207.009 
+0 0 20171207.010 
+0 0 20171207.011 
+0 0 20171207.012 
+0 0 20171207.013 
+0 0 20171207.014 
+0 0 20171207.016 
+0 0 20171207.017 
+0 0 20171207.018 
+0 0 20171207.019 
+0 0 20171207.020 
+0 0 20171207.021 
+0 0 20171207.022 
+0 0 20171207.023 
+0 0 20171207.024 
+0 0 20171207.025 
+0 0 20171207.026 
+0 0 20171207.027 
+0 0 20171207.028 
+0 0 20171207.029 
+0 0 20171207.030 
+0 0 20171207.031 
+0 0 20171207.032 
+0 0 20171207.033 
+0 0 20171207.034 
+0 0 20171207.035 
+0 0 20171207.037 
+0 0 20171207.038 
+0 0 20171207.039 
+0 0 20171207.040 
+0 0 20171207.041 
+0 0 20171207.042 
+0 0 20171207.043 
+0 0 20171207.044 
+0 0 20171207.045 
+0 0 20171207.046 
+0 0 20171207.047 
+0 0 20171207.048 
+0 0 20171207.049 
+0 0 20171207.050 
+0 0 20171207.051 
diff --git a/AEF30_coldframes_background_fails_real.txt b/AEF30_coldframes_background_fails_real.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ebc17a65228371f630ce291dfd0e1ab1f208e68b
--- /dev/null
+++ b/AEF30_coldframes_background_fails_real.txt
@@ -0,0 +1,166 @@
+coldframes_fail_shots coldframe_value background_fail_shots ; numberofPrograms=946 ; numberoffails=164 
+20170926.009 0.0222826562296 20170926.010 
+0 0 20170926.010 
+0 0 20170926.012 
+0 0 20170927.005 
+0 0 20170927.006 
+0 0 20170927.007 
+0 0 20170927.008 
+0 0 20170927.009 
+0 0 20170927.010 
+0 0 20170927.011 
+0 0 20171004.007 
+0 0 20171004.008 
+0 0 20171004.009 
+0 0 20171005.007 
+0 0 20171005.008 
+0 0 20171005.009 
+0 0 20171005.010 
+0 0 20171005.011 
+0 0 20171010.006 
+0 0 20171010.007 
+0 0 20171010.008 
+0 0 20171010.009 
+0 0 20171010.010 
+0 0 20171010.011 
+0 0 20171010.012 
+0 0 20171010.013 
+0 0 20171010.014 
+0 0 20171010.015 
+0 0 20171010.016 
+0 0 20171010.017 
+0 0 20171010.018 
+0 0 20171010.019 
+0 0 20171010.020 
+0 0 20171011.007 
+0 0 20171011.008 
+0 0 20171011.009 
+0 0 20171011.010 
+0 0 20171012.009 
+0 0 20171012.010 
+0 0 20171017.011 
+0 0 20171017.012 
+0 0 20171017.013 
+0 0 20171017.014 
+0 0 20171017.015 
+0 0 20171017.016 
+0 0 20171017.017 
+0 0 20171024.012 
+0 0 20171024.013 
+0 0 20171024.014 
+0 0 20171024.015 
+0 0 20171024.016 
+0 0 20171024.017 
+0 0 20171024.018 
+0 0 20171024.019 
+0 0 20171024.021 
+0 0 20171024.042 
+0 0 20171024.044 
+0 0 20171026.005 
+0 0 20171026.006 
+0 0 20171026.007 
+0 0 20171026.008 
+0 0 20171101.012 
+0 0 20171101.013 
+0 0 20171101.014 
+0 0 20171101.015 
+0 0 20171101.016 
+0 0 20171101.017 
+0 0 20171101.018 
+0 0 20171101.019 
+0 0 20171101.020 
+0 0 20171102.007 
+0 0 20171102.008 
+0 0 20171102.009 
+0 0 20171102.010 
+0 0 20171102.011 
+0 0 20171102.012 
+0 0 20171102.013 
+0 0 20171102.014 
+0 0 20171102.015 
+0 0 20171102.016 
+0 0 20171102.017 
+0 0 20171102.018 
+0 0 20171102.019 
+0 0 20171102.020 
+0 0 20171102.021 
+0 0 20171102.022 
+0 0 20171102.023 
+0 0 20171102.024 
+0 0 20171102.025 
+0 0 20171102.026 
+0 0 20171102.028 
+0 0 20171102.029 
+0 0 20171107.015 
+0 0 20171107.016 
+0 0 20171107.017 
+0 0 20171107.018 
+0 0 20171107.019 
+0 0 20171107.020 
+0 0 20171107.021 
+0 0 20171107.022 
+0 0 20171107.023 
+0 0 20171108.007 
+0 0 20171108.008 
+0 0 20171108.009 
+0 0 20171108.010 
+0 0 20171108.011 
+0 0 20171108.012 
+0 0 20171108.013 
+0 0 20171108.014 
+0 0 20171114.012 
+0 0 20171114.013 
+0 0 20171114.014 
+0 0 20171114.015 
+0 0 20171114.016 
+0 0 20171114.017 
+0 0 20171114.018 
+0 0 20171114.019 
+0 0 20171114.020 
+0 0 20171114.021 
+0 0 20171114.022 
+0 0 20171114.023 
+0 0 20171114.024 
+0 0 20171114.025 
+0 0 20171114.026 
+0 0 20171114.027 
+0 0 20171114.028 
+0 0 20171114.029 
+0 0 20171114.030 
+0 0 20171114.031 
+0 0 20171114.032 
+0 0 20171115.006 
+0 0 20171121.007 
+0 0 20171121.008 
+0 0 20171121.009 
+0 0 20171121.010 
+0 0 20171122.006 
+0 0 20171122.007 
+0 0 20171122.008 
+0 0 20171123.008 
+0 0 20171123.009 
+0 0 20171123.010 
+0 0 20171123.011 
+0 0 20171123.012 
+0 0 20171123.013 
+0 0 20171123.014 
+0 0 20171123.015 
+0 0 20171123.016 
+0 0 20171123.017 
+0 0 20171123.019 
+0 0 20171123.031 
+0 0 20171123.032 
+0 0 20171123.033 
+0 0 20171123.034 
+0 0 20171123.035 
+0 0 20171123.036 
+0 0 20171123.037 
+0 0 20171129.008 
+0 0 20171129.009 
+0 0 20171205.006 
+0 0 20171205.007 
+0 0 20171205.008 
+0 0 20171205.009 
+0 0 20171206.013 
+0 0 20171206.028 
+0 0 20171207.006 
diff --git a/AEF31_coldframes_background_fails_real.txt b/AEF31_coldframes_background_fails_real.txt
new file mode 100644
index 0000000000000000000000000000000000000000..67abbd4ad8ca78bb625c0b8f270c710866d1775e
--- /dev/null
+++ b/AEF31_coldframes_background_fails_real.txt
@@ -0,0 +1,947 @@
+coldframes_fail_shots coldframe_value background_fail_shots ; numberofPrograms=986 ; numberoffails=945 
+20170926.009 0.347553910489 20170926.010 
+20170926.029 0.485918635308 20170926.012 
+20170926.030 0.476594516749 20170926.014 
+20170926.034 0.489775778353 20170926.016 
+20170926.035 0.48105384234 20170926.017 
+20170926.036 0.47574668615 20170926.019 
+20170926.037 0.468420608294 20170926.021 
+20170926.038 0.465362095479 20170926.022 
+20170926.039 0.462117169713 20170926.024 
+20170926.040 0.458388197475 20170926.025 
+20170926.041 0.456201554508 20170926.026 
+20170926.042 0.457926881275 20170926.027 
+20170926.043 0.458104396551 20170926.028 
+20170926.044 0.45898340768 20170926.029 
+20170926.045 0.45964042492 20170926.030 
+20170926.046 0.461778750422 20170926.031 
+20170926.047 0.46302656355 20170926.032 
+20170926.048 0.462996970694 20170926.033 
+20170926.049 0.458491485634 20170926.034 
+20170926.050 0.456358482595 20170926.035 
+20170926.051 0.467718337181 20170926.036 
+20170926.052 0.466511467034 20170926.037 
+20171004.015 0.494905976587 20170926.038 
+20171004.016 0.468706493641 20170926.039 
+20171004.017 0.45221764148 20170926.040 
+20171004.018 0.441640135198 20170926.041 
+20171004.019 0.436197268875 20170926.042 
+20171004.020 0.43558057429 20170926.043 
+20171004.021 0.433378174471 20170926.044 
+20171004.022 0.433079961704 20170926.045 
+20171004.023 0.42583728445 20170926.046 
+20171004.024 0.424786833656 20170926.047 
+20171004.025 0.425147455513 20170926.048 
+20171004.026 0.425063324982 20170926.049 
+20171004.027 0.424778550048 20170926.050 
+20171004.028 0.426559723063 20170926.051 
+20171004.029 0.425746823199 20170926.052 
+20171004.030 0.425162750661 20170927.005 
+20171004.031 0.425632119203 20170927.006 
+20171004.032 0.426574295999 20170927.007 
+20171004.033 0.426170408346 20170927.008 
+20171004.034 0.425827694258 20170927.009 
+20171004.035 0.425182417453 20170927.010 
+20171004.036 0.426334026469 20170927.011 
+20171004.037 0.427467638296 20170927.012 
+20171004.038 0.425944517116 20170927.013 
+20171004.039 0.43065632919 20170927.014 
+20171004.040 0.43074901799 20170927.015 
+20171004.042 0.477660214264 20170927.016 
+20171004.043 0.48415575534 20170927.017 
+20171004.044 0.49270625292 20170927.018 
+20171011.018 0.488041526066 20170927.019 
+20171011.019 0.474166229537 20170927.020 
+20171011.020 0.471760252685 20170927.021 
+20171011.021 0.488758571731 20170927.022 
+20171011.022 0.483672607574 20170927.023 
+20171011.023 0.47386123052 20170927.024 
+20171011.024 0.484203384124 20170927.025 
+20171011.025 0.485263398887 20170927.026 
+20171011.026 0.48899427853 20170927.027 
+20171011.027 0.481055874574 20170927.028 
+20171011.028 0.49120101082 20170927.029 
+20171011.030 0.489029327334 20170927.030 
+20171011.031 0.477289217616 20170927.031 
+20171011.032 0.471593323717 20170927.032 
+20171011.034 0.495161514763 20170927.033 
+20171011.035 0.475783437457 20170927.034 
+20171011.036 0.460106534917 20170927.035 
+20171011.037 0.451989630518 20170927.036 
+20171011.038 0.44721013032 20170927.037 
+20171011.039 0.437948864387 20170927.038 
+20171011.040 0.45026489833 20170927.039 
+20171011.041 0.44601530441 20170927.040 
+20171011.042 0.44974705751 20170927.041 
+20171011.043 0.448547995999 20170927.042 
+20171011.044 0.453972475328 20171004.007 
+20171011.045 0.469439920433 20171004.008 
+20171011.046 0.466334708168 20171004.009 
+20171011.047 0.474873030087 20171004.010 
+20171011.048 0.479759241608 20171004.011 
+20171011.049 0.475311293529 20171004.012 
+20171011.050 0.477967610701 20171004.013 
+20171011.051 0.479156570537 20171004.014 
+20171011.052 0.47495607936 20171004.015 
+20171011.053 0.471671602471 20171004.016 
+20171011.054 0.467400910054 20171004.017 
+20171011.055 0.46275066987 20171004.018 
+20171012.026 0.499294699025 20171004.019 
+20171012.027 0.491184790619 20171004.020 
+20171012.028 0.493031099474 20171004.021 
+20171012.029 0.493054973145 20171004.022 
+20171012.036 0.49795911397 20171004.023 
+20171012.037 0.474454554998 20171004.024 
+20171012.038 0.466745212656 20171004.025 
+20171012.039 0.462576497346 20171004.026 
+20171012.040 0.460056458547 20171004.027 
+20171012.041 0.456692230641 20171004.028 
+20171012.042 0.471723010165 20171004.029 
+20171017.032 0.481719907233 20171004.030 
+20171017.033 0.498148496541 20171004.031 
+20171017.034 0.488872159604 20171004.032 
+20171017.035 0.47698498314 20171004.033 
+20171017.036 0.475643875405 20171004.034 
+20171017.037 0.477289990611 20171004.035 
+20171017.038 0.478568684272 20171004.036 
+20171017.039 0.473690593744 20171004.037 
+20171017.040 0.479649504914 20171004.038 
+20171017.055 0.498074729825 20171004.039 
+20171018.015 0.489050714734 20171004.040 
+20171018.016 0.498153104532 20171004.042 
+20171018.017 0.491477853762 20171004.043 
+20171018.018 0.480632037274 20171004.044 
+20171018.019 0.482094164471 20171004.046 
+20171018.020 0.473970868918 20171004.047 
+20171018.021 0.468935096954 20171005.007 
+20171018.032 0.490878797945 20171005.008 
+20171018.033 0.478725238328 20171005.009 
+20171018.034 0.475188645869 20171005.010 
+20171018.035 0.462800515933 20171005.011 
+20171018.036 0.45557553735 20171005.012 
+20171018.037 0.448883670276 20171005.013 
+20171018.038 0.444050418519 20171005.014 
+20171018.039 0.442502583138 20171005.015 
+20171018.040 0.471567938856 20171005.016 
+20171018.041 0.486815083234 20171005.017 
+20171018.042 0.494998962226 20171005.018 
+20171018.043 0.492685089013 20171005.019 
+20171018.044 0.493899207888 20171005.020 
+20171018.045 0.493073326525 20171005.021 
+20171018.046 0.489438553214 20171005.022 
+20171026.011 0.480138076696 20171005.023 
+20171026.012 0.443626985764 20171005.024 
+20171026.013 0.472253490205 20171005.025 
+20171026.014 0.485865591266 20171005.026 
+20171026.015 0.498552600029 20171005.027 
+20171026.022 0.493400021633 20171005.028 
+20171026.024 0.487160782095 20171005.029 
+20171026.025 0.473112266654 20171005.030 
+20171026.026 0.472715248131 20171005.031 
+20171026.027 0.483826933628 20171005.032 
+20171026.028 0.469713722535 20171005.033 
+20171026.029 0.459385381694 20171005.034 
+20171026.030 0.473668449891 20171005.035 
+20171026.031 0.485450561573 20171005.036 
+20171026.032 0.466881342335 20171005.037 
+20171026.033 0.451169075581 20171005.038 
+20171026.034 0.445624452805 20171005.039 
+20171026.035 0.449469518453 20171005.040 
+20171026.036 0.44782105974 20171005.041 
+20171026.037 0.454981089859 20171005.042 
+20171026.038 0.446098110653 20171005.043 
+20171026.039 0.451230930833 20171005.044 
+20171026.040 0.444708906169 20171005.045 
+20171026.041 0.443679413101 20171005.046 
+20171101.027 0.4940620695 20171005.047 
+20171101.028 0.476536488314 20171010.006 
+20171101.029 0.468209772478 20171010.007 
+20171101.030 0.472741969542 20171010.008 
+20171101.031 0.483989618484 20171010.009 
+20171101.032 0.499356186472 20171010.010 
+20171102.041 0.49573388644 20171010.011 
+20171102.043 0.498152022638 20171010.012 
+20171102.044 0.477937812326 20171010.013 
+20171102.045 0.466299687159 20171010.014 
+20171102.046 0.453910571942 20171010.015 
+20171102.047 0.461090275767 20171010.016 
+20171102.048 0.453564915048 20171010.017 
+20171102.049 0.451391294505 20171010.018 
+20171102.050 0.460688680617 20171010.019 
+20171108.017 0.482962240743 20171010.020 
+20171108.018 0.458208929745 20171010.021 
+20171108.019 0.453413375804 20171010.022 
+20171108.020 0.442194010415 20171010.023 
+20171108.021 0.431860252745 20171010.024 
+20171108.022 0.426597657386 20171010.025 
+20171108.023 0.432697913875 20171010.026 
+20171108.024 0.444829272175 20171010.027 
+20171108.025 0.447860165538 20171010.028 
+20171108.026 0.443933372621 20171010.029 
+20171108.027 0.447752237405 20171010.030 
+20171108.028 0.45619727257 20171010.031 
+20171108.029 0.457290794434 20171011.007 
+20171108.030 0.460011662953 20171011.008 
+20171108.038 0.482905983905 20171011.009 
+20171108.039 0.488161970609 20171011.010 
+20171108.040 0.477221061803 20171011.011 
+20171108.043 0.43700196966 20171011.012 
+20171108.044 0.457707463069 20171011.013 
+20171108.045 0.469094084025 20171011.014 
+20171108.046 0.474264977977 20171011.015 
+20171109.014 0.488794997791 20171011.016 
+20171109.015 0.481146552432 20171011.017 
+20171109.016 0.477029952392 20171011.018 
+20171109.017 0.456938033852 20171011.019 
+20171109.018 0.444844835503 20171011.020 
+20171109.019 0.440285003704 20171011.021 
+20171109.020 0.442185763195 20171011.022 
+20171109.021 0.434354772883 20171011.023 
+20171109.022 0.430039736432 20171011.024 
+20171109.023 0.428802624299 20171011.025 
+20171109.024 0.428355875195 20171011.026 
+20171109.025 0.429745409184 20171011.027 
+20171109.026 0.42981035987 20171011.028 
+20171109.027 0.430076544591 20171011.029 
+20171109.028 0.430522345527 20171011.030 
+20171109.029 0.431966857861 20171011.031 
+20171109.030 0.433336914997 20171011.032 
+20171109.031 0.436631410998 20171011.033 
+20171109.032 0.441946526805 20171011.034 
+20171109.033 0.432839820387 20171011.035 
+20171109.034 0.429556042666 20171011.036 
+20171109.035 0.427626864129 20171011.037 
+20171109.036 0.426766791834 20171011.038 
+20171109.037 0.427669390371 20171011.039 
+20171109.038 0.429151783952 20171011.040 
+20171109.039 0.427399258141 20171011.041 
+20171109.040 0.427107366256 20171011.042 
+20171109.041 0.42881870874 20171011.043 
+20171109.042 0.428238310601 20171011.044 
+20171109.043 0.428274968523 20171011.045 
+20171109.044 0.428175971579 20171011.046 
+20171109.045 0.427682754554 20171011.047 
+20171109.046 0.427913831372 20171011.048 
+20171109.047 0.429817471619 20171011.049 
+20171109.048 0.430030493124 20171011.050 
+20171109.049 0.436346566485 20171011.051 
+20171109.050 0.45087787799 20171011.052 
+20171109.051 0.457760446231 20171011.053 
+20171109.052 0.458014129256 20171011.054 
+20171109.053 0.465495914316 20171011.055 
+20171109.054 0.486765647885 20171012.009 
+20171109.055 0.487672807876 20171012.010 
+20171114.035 0.479883635369 20171012.011 
+20171114.036 0.442441191498 20171012.012 
+20171114.037 0.461327093104 20171012.013 
+20171114.038 0.475759793345 20171012.014 
+20171114.040 0.480678967483 20171012.015 
+20171114.041 0.478282725619 20171012.016 
+20171114.042 0.494084800706 20171012.017 
+20171114.046 0.43716023749 20171012.018 
+20171114.047 0.446981229648 20171012.019 
+20171114.048 0.442939793493 20171012.020 
+20171114.049 0.434406299295 20171012.021 
+20171114.050 0.439453927235 20171012.022 
+20171114.051 0.446169126299 20171012.023 
+20171114.052 0.453829212342 20171012.024 
+20171114.053 0.451594963464 20171012.025 
+20171114.054 0.447343776719 20171012.026 
+20171115.009 0.469494089297 20171012.027 
+20171115.010 0.430864559639 20171012.028 
+20171115.011 0.428391984272 20171012.029 
+20171115.012 0.422547069512 20171012.030 
+20171115.013 0.424300544321 20171012.031 
+20171115.014 0.423115295467 20171012.032 
+20171115.015 0.429935268476 20171012.033 
+20171115.016 0.433420121204 20171012.034 
+20171115.017 0.433287180407 20171012.035 
+20171115.018 0.428214454488 20171012.036 
+20171115.019 0.427670670021 20171012.037 
+20171115.020 0.429948469566 20171012.038 
+20171115.021 0.432423113565 20171012.039 
+20171115.022 0.435633978071 20171012.040 
+20171115.023 0.438421771758 20171012.041 
+20171115.024 0.441526438071 20171012.042 
+20171115.025 0.439970735582 20171012.043 
+20171115.026 0.429779008184 20171012.045 
+20171115.028 0.428808186331 20171012.046 
+20171115.030 0.431491579965 20171012.047 
+20171115.031 0.43122251818 20171012.048 
+20171115.032 0.431618118528 20171012.049 
+20171115.033 0.432299952261 20171012.050 
+20171115.034 0.436051617422 20171012.051 
+20171115.035 0.43024370845 20171012.052 
+20171115.036 0.429426263018 20171012.053 
+20171115.037 0.432261974896 20171012.054 
+20171115.038 0.436731725163 20171012.055 
+20171115.039 0.441077087143 20171017.011 
+20171121.013 0.492968292093 20171017.012 
+20171121.014 0.439920651682 20171017.013 
+20171121.015 0.4364002975 20171017.014 
+20171121.016 0.429001870609 20171017.015 
+20171121.017 0.431047682529 20171017.016 
+20171121.018 0.42640596483 20171017.017 
+20171121.019 0.421724963731 20171017.018 
+20171121.020 0.423294579393 20171017.019 
+20171121.021 0.431465511872 20171017.020 
+20171121.023 0.432030444326 20171017.021 
+20171121.024 0.431555479139 20171017.022 
+20171121.025 0.432503182016 20171017.023 
+20171121.026 0.431491532648 20171017.024 
+20171121.027 0.435944815561 20171017.025 
+20171121.028 0.440689456081 20171017.026 
+20171121.030 0.432094564813 20171017.027 
+20171121.036 0.432828717662 20171017.028 
+20171121.040 0.432377380277 20171017.029 
+20171122.019 0.489706881569 20171017.030 
+20171122.020 0.456318026583 20171017.031 
+20171122.021 0.468350895667 20171017.032 
+20171122.022 0.466633409389 20171017.033 
+20171122.023 0.470246797865 20171017.034 
+20171122.024 0.458493512357 20171017.035 
+20171122.025 0.458778331126 20171017.036 
+20171122.026 0.492880855519 20171017.037 
+20171122.027 0.477373238099 20171017.038 
+20171122.028 0.482537270075 20171017.039 
+20171122.032 0.49538750668 20171017.040 
+20171122.033 0.493342946576 20171017.041 
+20171122.034 0.494962739123 20171017.042 
+20171122.035 0.474322055777 20171017.043 
+20171122.036 0.443437749413 20171017.044 
+20171129.015 0.487176729106 20171017.045 
+20171129.016 0.453631050637 20171017.046 
+20171129.017 0.469223773665 20171017.047 
+20171129.018 0.419936663899 20171017.048 
+20171129.019 0.428580769105 20171017.049 
+20171129.020 0.43716639279 20171017.050 
+20171129.021 0.459610921435 20171017.051 
+20171129.022 0.44608754822 20171017.052 
+20171129.023 0.440921092956 20171017.053 
+20171129.025 0.446744716707 20171017.054 
+20171129.026 0.444894678664 20171017.055 
+20171129.027 0.442995540454 20171018.005 
+20171129.028 0.451135365526 20171018.006 
+20171129.029 0.444863510025 20171018.007 
+20171129.030 0.425873251272 20171018.008 
+20171129.031 0.411074884986 20171018.009 
+20171129.032 0.437796483773 20171018.010 
+20171129.033 0.436596157728 20171018.011 
+20171129.034 0.436467601748 20171018.012 
+20171129.035 0.435490298458 20171018.013 
+20171129.036 0.43132307875 20171018.014 
+20171129.037 0.070815677134 20171018.015 
+20171129.038 0.431586959922 20171018.016 
+20171129.039 0.423372757252 20171018.017 
+20171129.040 0.421344270925 20171018.018 
+20171129.041 0.418768018329 20171018.019 
+20171129.042 0.416724878551 20171018.020 
+20171129.043 0.43075027682 20171018.021 
+20171129.044 0.427552720456 20171018.022 
+20171205.025 0.481243146729 20171018.023 
+20171205.026 0.452707376789 20171018.024 
+20171205.027 0.466341589052 20171018.025 
+20171206.024 0.490072464945 20171018.026 
+0 0 20171018.026 
+0 0 20171018.027 
+0 0 20171018.028 
+0 0 20171018.029 
+0 0 20171018.030 
+0 0 20171018.031 
+0 0 20171018.032 
+0 0 20171018.033 
+0 0 20171018.034 
+0 0 20171018.035 
+0 0 20171018.036 
+0 0 20171018.037 
+0 0 20171018.038 
+0 0 20171018.039 
+0 0 20171018.040 
+0 0 20171018.041 
+0 0 20171018.042 
+0 0 20171018.043 
+0 0 20171018.044 
+0 0 20171018.045 
+0 0 20171018.046 
+0 0 20171024.012 
+0 0 20171024.013 
+0 0 20171024.014 
+0 0 20171024.015 
+0 0 20171024.016 
+0 0 20171024.017 
+0 0 20171024.018 
+0 0 20171024.019 
+0 0 20171024.020 
+0 0 20171024.021 
+0 0 20171024.023 
+0 0 20171024.024 
+0 0 20171024.025 
+0 0 20171024.026 
+0 0 20171024.027 
+0 0 20171024.028 
+0 0 20171024.029 
+0 0 20171024.030 
+0 0 20171024.031 
+0 0 20171024.032 
+0 0 20171024.033 
+0 0 20171024.034 
+0 0 20171024.035 
+0 0 20171024.036 
+0 0 20171024.037 
+0 0 20171024.038 
+0 0 20171024.039 
+0 0 20171024.040 
+0 0 20171024.041 
+0 0 20171024.042 
+0 0 20171024.043 
+0 0 20171024.044 
+0 0 20171024.045 
+0 0 20171024.046 
+0 0 20171024.047 
+0 0 20171024.048 
+0 0 20171024.049 
+0 0 20171024.050 
+0 0 20171024.051 
+0 0 20171024.052 
+0 0 20171024.053 
+0 0 20171024.054 
+0 0 20171025.006 
+0 0 20171025.007 
+0 0 20171025.008 
+0 0 20171025.009 
+0 0 20171025.010 
+0 0 20171025.011 
+0 0 20171025.012 
+0 0 20171025.013 
+0 0 20171025.014 
+0 0 20171025.015 
+0 0 20171025.016 
+0 0 20171025.017 
+0 0 20171025.018 
+0 0 20171025.019 
+0 0 20171025.020 
+0 0 20171025.021 
+0 0 20171025.022 
+0 0 20171025.023 
+0 0 20171025.024 
+0 0 20171025.025 
+0 0 20171025.026 
+0 0 20171025.027 
+0 0 20171025.028 
+0 0 20171025.029 
+0 0 20171025.030 
+0 0 20171025.031 
+0 0 20171025.032 
+0 0 20171025.033 
+0 0 20171025.034 
+0 0 20171025.035 
+0 0 20171025.036 
+0 0 20171025.037 
+0 0 20171025.038 
+0 0 20171025.039 
+0 0 20171025.040 
+0 0 20171025.041 
+0 0 20171025.042 
+0 0 20171025.043 
+0 0 20171025.044 
+0 0 20171025.045 
+0 0 20171025.046 
+0 0 20171025.047 
+0 0 20171025.048 
+0 0 20171025.049 
+0 0 20171025.050 
+0 0 20171025.051 
+0 0 20171025.052 
+0 0 20171025.054 
+0 0 20171025.055 
+0 0 20171026.005 
+0 0 20171026.006 
+0 0 20171026.007 
+0 0 20171026.008 
+0 0 20171026.009 
+0 0 20171026.010 
+0 0 20171026.011 
+0 0 20171026.012 
+0 0 20171026.013 
+0 0 20171026.014 
+0 0 20171026.015 
+0 0 20171026.016 
+0 0 20171026.017 
+0 0 20171026.018 
+0 0 20171026.019 
+0 0 20171026.020 
+0 0 20171026.021 
+0 0 20171026.022 
+0 0 20171026.023 
+0 0 20171026.024 
+0 0 20171026.025 
+0 0 20171026.026 
+0 0 20171026.027 
+0 0 20171026.028 
+0 0 20171026.029 
+0 0 20171026.030 
+0 0 20171026.031 
+0 0 20171026.032 
+0 0 20171026.033 
+0 0 20171026.034 
+0 0 20171026.035 
+0 0 20171026.036 
+0 0 20171026.037 
+0 0 20171026.038 
+0 0 20171026.039 
+0 0 20171026.040 
+0 0 20171026.041 
+0 0 20171101.012 
+0 0 20171101.013 
+0 0 20171101.014 
+0 0 20171101.015 
+0 0 20171101.016 
+0 0 20171101.017 
+0 0 20171101.018 
+0 0 20171101.019 
+0 0 20171101.020 
+0 0 20171101.021 
+0 0 20171101.022 
+0 0 20171101.023 
+0 0 20171101.024 
+0 0 20171101.025 
+0 0 20171101.026 
+0 0 20171101.027 
+0 0 20171101.028 
+0 0 20171101.029 
+0 0 20171101.030 
+0 0 20171101.031 
+0 0 20171101.032 
+0 0 20171101.033 
+0 0 20171101.034 
+0 0 20171101.035 
+0 0 20171101.036 
+0 0 20171101.037 
+0 0 20171101.038 
+0 0 20171101.039 
+0 0 20171101.040 
+0 0 20171102.007 
+0 0 20171102.008 
+0 0 20171102.009 
+0 0 20171102.010 
+0 0 20171102.011 
+0 0 20171102.012 
+0 0 20171102.013 
+0 0 20171102.014 
+0 0 20171102.015 
+0 0 20171102.016 
+0 0 20171102.017 
+0 0 20171102.018 
+0 0 20171102.019 
+0 0 20171102.020 
+0 0 20171102.021 
+0 0 20171102.022 
+0 0 20171102.023 
+0 0 20171102.024 
+0 0 20171102.025 
+0 0 20171102.026 
+0 0 20171102.027 
+0 0 20171102.028 
+0 0 20171102.029 
+0 0 20171102.030 
+0 0 20171102.031 
+0 0 20171102.032 
+0 0 20171102.033 
+0 0 20171102.034 
+0 0 20171102.035 
+0 0 20171102.036 
+0 0 20171102.037 
+0 0 20171102.038 
+0 0 20171102.039 
+0 0 20171102.040 
+0 0 20171102.041 
+0 0 20171102.042 
+0 0 20171102.043 
+0 0 20171102.044 
+0 0 20171102.045 
+0 0 20171102.046 
+0 0 20171102.047 
+0 0 20171102.048 
+0 0 20171102.049 
+0 0 20171102.050 
+0 0 20171107.015 
+0 0 20171107.016 
+0 0 20171107.017 
+0 0 20171107.018 
+0 0 20171107.019 
+0 0 20171107.020 
+0 0 20171107.021 
+0 0 20171107.022 
+0 0 20171107.023 
+0 0 20171107.024 
+0 0 20171107.025 
+0 0 20171107.026 
+0 0 20171107.027 
+0 0 20171107.028 
+0 0 20171107.029 
+0 0 20171107.030 
+0 0 20171107.031 
+0 0 20171107.032 
+0 0 20171107.033 
+0 0 20171107.034 
+0 0 20171107.035 
+0 0 20171107.036 
+0 0 20171107.037 
+0 0 20171107.038 
+0 0 20171107.039 
+0 0 20171107.040 
+0 0 20171107.041 
+0 0 20171107.042 
+0 0 20171107.043 
+0 0 20171107.044 
+0 0 20171107.045 
+0 0 20171107.046 
+0 0 20171107.047 
+0 0 20171107.048 
+0 0 20171107.049 
+0 0 20171107.050 
+0 0 20171107.051 
+0 0 20171107.052 
+0 0 20171107.053 
+0 0 20171107.054 
+0 0 20171107.055 
+0 0 20171108.007 
+0 0 20171108.008 
+0 0 20171108.009 
+0 0 20171108.010 
+0 0 20171108.011 
+0 0 20171108.012 
+0 0 20171108.013 
+0 0 20171108.014 
+0 0 20171108.015 
+0 0 20171108.016 
+0 0 20171108.017 
+0 0 20171108.018 
+0 0 20171108.019 
+0 0 20171108.020 
+0 0 20171108.021 
+0 0 20171108.022 
+0 0 20171108.023 
+0 0 20171108.024 
+0 0 20171108.025 
+0 0 20171108.026 
+0 0 20171108.027 
+0 0 20171108.028 
+0 0 20171108.029 
+0 0 20171108.030 
+0 0 20171108.031 
+0 0 20171108.032 
+0 0 20171108.033 
+0 0 20171108.034 
+0 0 20171108.035 
+0 0 20171108.038 
+0 0 20171108.039 
+0 0 20171108.040 
+0 0 20171108.043 
+0 0 20171108.044 
+0 0 20171108.045 
+0 0 20171108.046 
+0 0 20171109.005 
+0 0 20171109.006 
+0 0 20171109.007 
+0 0 20171109.008 
+0 0 20171109.009 
+0 0 20171109.010 
+0 0 20171109.011 
+0 0 20171109.012 
+0 0 20171109.013 
+0 0 20171109.014 
+0 0 20171109.015 
+0 0 20171109.016 
+0 0 20171109.017 
+0 0 20171109.018 
+0 0 20171109.019 
+0 0 20171109.020 
+0 0 20171109.021 
+0 0 20171109.022 
+0 0 20171109.023 
+0 0 20171109.024 
+0 0 20171109.025 
+0 0 20171109.026 
+0 0 20171109.027 
+0 0 20171109.028 
+0 0 20171109.029 
+0 0 20171109.030 
+0 0 20171109.031 
+0 0 20171109.032 
+0 0 20171109.033 
+0 0 20171109.034 
+0 0 20171109.035 
+0 0 20171109.036 
+0 0 20171109.037 
+0 0 20171109.038 
+0 0 20171109.039 
+0 0 20171109.040 
+0 0 20171109.041 
+0 0 20171109.042 
+0 0 20171109.043 
+0 0 20171109.044 
+0 0 20171109.045 
+0 0 20171109.046 
+0 0 20171109.047 
+0 0 20171109.048 
+0 0 20171109.049 
+0 0 20171109.050 
+0 0 20171109.051 
+0 0 20171109.052 
+0 0 20171109.053 
+0 0 20171109.054 
+0 0 20171109.055 
+0 0 20171114.012 
+0 0 20171114.013 
+0 0 20171114.014 
+0 0 20171114.015 
+0 0 20171114.016 
+0 0 20171114.017 
+0 0 20171114.018 
+0 0 20171114.019 
+0 0 20171114.020 
+0 0 20171114.021 
+0 0 20171114.022 
+0 0 20171114.023 
+0 0 20171114.024 
+0 0 20171114.025 
+0 0 20171114.026 
+0 0 20171114.027 
+0 0 20171114.028 
+0 0 20171114.029 
+0 0 20171114.030 
+0 0 20171114.031 
+0 0 20171114.032 
+0 0 20171114.033 
+0 0 20171114.034 
+0 0 20171114.035 
+0 0 20171114.036 
+0 0 20171114.037 
+0 0 20171114.038 
+0 0 20171114.039 
+0 0 20171114.040 
+0 0 20171114.041 
+0 0 20171114.042 
+0 0 20171114.043 
+0 0 20171114.046 
+0 0 20171114.047 
+0 0 20171114.048 
+0 0 20171114.049 
+0 0 20171114.050 
+0 0 20171114.051 
+0 0 20171114.052 
+0 0 20171114.053 
+0 0 20171114.054 
+0 0 20171115.006 
+0 0 20171115.007 
+0 0 20171115.008 
+0 0 20171115.009 
+0 0 20171115.010 
+0 0 20171115.011 
+0 0 20171115.012 
+0 0 20171115.013 
+0 0 20171115.014 
+0 0 20171115.015 
+0 0 20171115.016 
+0 0 20171115.017 
+0 0 20171115.018 
+0 0 20171115.019 
+0 0 20171115.020 
+0 0 20171115.021 
+0 0 20171115.022 
+0 0 20171115.023 
+0 0 20171115.024 
+0 0 20171115.025 
+0 0 20171115.026 
+0 0 20171115.028 
+0 0 20171115.030 
+0 0 20171115.031 
+0 0 20171115.032 
+0 0 20171115.033 
+0 0 20171115.034 
+0 0 20171115.035 
+0 0 20171115.036 
+0 0 20171115.037 
+0 0 20171115.038 
+0 0 20171115.039 
+0 0 20171121.007 
+0 0 20171121.008 
+0 0 20171121.009 
+0 0 20171121.010 
+0 0 20171121.011 
+0 0 20171121.012 
+0 0 20171121.013 
+0 0 20171121.014 
+0 0 20171121.015 
+0 0 20171121.016 
+0 0 20171121.017 
+0 0 20171121.018 
+0 0 20171121.019 
+0 0 20171121.020 
+0 0 20171121.021 
+0 0 20171121.023 
+0 0 20171121.024 
+0 0 20171121.025 
+0 0 20171121.026 
+0 0 20171121.027 
+0 0 20171121.028 
+0 0 20171121.030 
+0 0 20171121.036 
+0 0 20171121.040 
+0 0 20171122.006 
+0 0 20171122.007 
+0 0 20171122.008 
+0 0 20171122.009 
+0 0 20171122.010 
+0 0 20171122.011 
+0 0 20171122.012 
+0 0 20171122.013 
+0 0 20171122.014 
+0 0 20171122.015 
+0 0 20171122.016 
+0 0 20171122.017 
+0 0 20171122.018 
+0 0 20171122.019 
+0 0 20171122.020 
+0 0 20171122.021 
+0 0 20171122.022 
+0 0 20171122.023 
+0 0 20171122.024 
+0 0 20171122.025 
+0 0 20171122.026 
+0 0 20171122.027 
+0 0 20171122.028 
+0 0 20171122.029 
+0 0 20171122.030 
+0 0 20171122.031 
+0 0 20171122.032 
+0 0 20171122.033 
+0 0 20171122.034 
+0 0 20171122.035 
+0 0 20171122.036 
+0 0 20171122.037 
+0 0 20171122.038 
+0 0 20171122.039 
+0 0 20171123.008 
+0 0 20171123.009 
+0 0 20171123.010 
+0 0 20171123.011 
+0 0 20171123.012 
+0 0 20171123.013 
+0 0 20171123.014 
+0 0 20171123.015 
+0 0 20171123.016 
+0 0 20171123.017 
+0 0 20171123.018 
+0 0 20171123.019 
+0 0 20171123.021 
+0 0 20171123.022 
+0 0 20171123.023 
+0 0 20171123.024 
+0 0 20171123.025 
+0 0 20171123.026 
+0 0 20171123.027 
+0 0 20171123.028 
+0 0 20171123.031 
+0 0 20171123.032 
+0 0 20171123.033 
+0 0 20171123.034 
+0 0 20171123.035 
+0 0 20171123.036 
+0 0 20171123.037 
+0 0 20171123.039 
+0 0 20171123.040 
+0 0 20171129.008 
+0 0 20171129.009 
+0 0 20171129.010 
+0 0 20171129.011 
+0 0 20171129.013 
+0 0 20171129.014 
+0 0 20171129.015 
+0 0 20171129.016 
+0 0 20171129.017 
+0 0 20171129.018 
+0 0 20171129.019 
+0 0 20171129.020 
+0 0 20171129.021 
+0 0 20171129.022 
+0 0 20171129.023 
+0 0 20171129.025 
+0 0 20171129.026 
+0 0 20171129.027 
+0 0 20171129.028 
+0 0 20171129.029 
+0 0 20171129.030 
+0 0 20171129.031 
+0 0 20171129.032 
+0 0 20171129.033 
+0 0 20171129.034 
+0 0 20171129.035 
+0 0 20171129.036 
+0 0 20171129.038 
+0 0 20171129.039 
+0 0 20171129.040 
+0 0 20171129.041 
+0 0 20171129.042 
+0 0 20171129.043 
+0 0 20171129.044 
+0 0 20171205.006 
+0 0 20171205.007 
+0 0 20171205.008 
+0 0 20171205.009 
+0 0 20171205.010 
+0 0 20171205.011 
+0 0 20171205.012 
+0 0 20171205.013 
+0 0 20171205.014 
+0 0 20171205.015 
+0 0 20171205.016 
+0 0 20171205.017 
+0 0 20171205.018 
+0 0 20171205.019 
+0 0 20171205.020 
+0 0 20171205.021 
+0 0 20171205.022 
+0 0 20171205.023 
+0 0 20171205.024 
+0 0 20171206.013 
+0 0 20171206.014 
+0 0 20171206.015 
+0 0 20171206.016 
+0 0 20171206.017 
+0 0 20171206.018 
+0 0 20171206.019 
+0 0 20171206.020 
+0 0 20171206.021 
+0 0 20171206.023 
+0 0 20171206.024 
+0 0 20171206.025 
+0 0 20171206.026 
+0 0 20171206.027 
+0 0 20171206.028 
+0 0 20171206.036 
+0 0 20171206.039 
+0 0 20171206.041 
+0 0 20171206.043 
+0 0 20171207.006 
+0 0 20171207.007 
+0 0 20171207.008 
+0 0 20171207.010 
+0 0 20171207.016 
+0 0 20171207.022 
+0 0 20171207.023 
+0 0 20171207.024 
+0 0 20171207.025 
+0 0 20171207.026 
+0 0 20171207.027 
+0 0 20171207.028 
+0 0 20171207.029 
+0 0 20171207.030 
+0 0 20171207.031 
+0 0 20171207.032 
+0 0 20171207.033 
+0 0 20171207.034 
+0 0 20171207.035 
+0 0 20171207.037 
+0 0 20171207.038 
diff --git a/AEF40_coldframes_background_fails_real.txt b/AEF40_coldframes_background_fails_real.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2671a61a7aa5589693c32de5a54421e7a03dc2a7
--- /dev/null
+++ b/AEF40_coldframes_background_fails_real.txt
@@ -0,0 +1,135 @@
+coldframes_fail_shots coldframe_value background_fail_shots ; numberofPrograms=951 ; numberoffails=133 
+20170926.009 0.772098602748 20170926.009 
+20171018.027 0.107558830475 20170926.010 
+20171205.022 0.728730946702 20170926.012 
+0 0 20170926.012 
+0 0 20170927.005 
+0 0 20170927.006 
+0 0 20170927.007 
+0 0 20170927.008 
+0 0 20171004.007 
+0 0 20171004.008 
+0 0 20171004.009 
+0 0 20171005.007 
+0 0 20171005.008 
+0 0 20171005.009 
+0 0 20171005.010 
+0 0 20171005.011 
+0 0 20171010.006 
+0 0 20171010.007 
+0 0 20171010.008 
+0 0 20171010.009 
+0 0 20171010.010 
+0 0 20171010.011 
+0 0 20171010.012 
+0 0 20171010.013 
+0 0 20171010.014 
+0 0 20171010.015 
+0 0 20171010.016 
+0 0 20171010.017 
+0 0 20171010.018 
+0 0 20171010.019 
+0 0 20171010.020 
+0 0 20171011.007 
+0 0 20171011.008 
+0 0 20171011.009 
+0 0 20171011.010 
+0 0 20171012.009 
+0 0 20171012.010 
+0 0 20171017.011 
+0 0 20171017.012 
+0 0 20171017.013 
+0 0 20171017.014 
+0 0 20171017.015 
+0 0 20171017.016 
+0 0 20171017.017 
+0 0 20171017.018 
+0 0 20171024.012 
+0 0 20171024.013 
+0 0 20171024.014 
+0 0 20171024.015 
+0 0 20171024.016 
+0 0 20171024.042 
+0 0 20171026.005 
+0 0 20171026.006 
+0 0 20171026.007 
+0 0 20171026.008 
+0 0 20171101.012 
+0 0 20171101.013 
+0 0 20171101.014 
+0 0 20171101.015 
+0 0 20171101.018 
+0 0 20171101.019 
+0 0 20171102.007 
+0 0 20171102.008 
+0 0 20171102.009 
+0 0 20171102.015 
+0 0 20171102.017 
+0 0 20171107.015 
+0 0 20171107.016 
+0 0 20171107.017 
+0 0 20171107.018 
+0 0 20171107.019 
+0 0 20171107.020 
+0 0 20171107.021 
+0 0 20171108.007 
+0 0 20171108.008 
+0 0 20171108.009 
+0 0 20171108.010 
+0 0 20171108.011 
+0 0 20171108.012 
+0 0 20171108.013 
+0 0 20171114.012 
+0 0 20171114.013 
+0 0 20171114.014 
+0 0 20171114.015 
+0 0 20171114.016 
+0 0 20171114.017 
+0 0 20171114.018 
+0 0 20171114.019 
+0 0 20171114.020 
+0 0 20171114.021 
+0 0 20171114.022 
+0 0 20171114.023 
+0 0 20171114.024 
+0 0 20171114.025 
+0 0 20171114.026 
+0 0 20171114.027 
+0 0 20171114.028 
+0 0 20171114.029 
+0 0 20171114.030 
+0 0 20171114.031 
+0 0 20171115.006 
+0 0 20171121.007 
+0 0 20171121.008 
+0 0 20171121.009 
+0 0 20171121.010 
+0 0 20171122.006 
+0 0 20171122.007 
+0 0 20171122.008 
+0 0 20171123.008 
+0 0 20171123.009 
+0 0 20171123.010 
+0 0 20171123.011 
+0 0 20171123.012 
+0 0 20171123.013 
+0 0 20171123.014 
+0 0 20171123.015 
+0 0 20171123.016 
+0 0 20171123.017 
+0 0 20171123.031 
+0 0 20171123.032 
+0 0 20171123.033 
+0 0 20171123.034 
+0 0 20171123.035 
+0 0 20171123.036 
+0 0 20171123.037 
+0 0 20171129.008 
+0 0 20171129.009 
+0 0 20171205.006 
+0 0 20171205.007 
+0 0 20171205.008 
+0 0 20171206.013 
+0 0 20171206.014 
+0 0 20171206.015 
+0 0 20171207.006 
diff --git a/AEF41_coldframes_background_fails_real.txt b/AEF41_coldframes_background_fails_real.txt
new file mode 100644
index 0000000000000000000000000000000000000000..22592e946fb4ccb250d1c30ad4102bdf16d97aeb
--- /dev/null
+++ b/AEF41_coldframes_background_fails_real.txt
@@ -0,0 +1,72 @@
+coldframes_fail_shots coldframe_value background_fail_shots ; numberofPrograms=920 ; numberoffails=70 
+20170926.009 0.152353670339 20170926.016 
+20170927.039 0.0536798150623 20170926.021 
+20171024.026 0.363698394271 20170926.022 
+20171024.027 0.330940144954 20170926.024 
+20171129.037 0.217471694264 20170926.025 
+0 0 20170926.025 
+0 0 20170927.005 
+0 0 20170927.007 
+0 0 20170927.008 
+0 0 20170927.009 
+0 0 20170927.010 
+0 0 20171005.007 
+0 0 20171005.008 
+0 0 20171005.009 
+0 0 20171005.010 
+0 0 20171010.006 
+0 0 20171010.007 
+0 0 20171011.007 
+0 0 20171017.011 
+0 0 20171017.012 
+0 0 20171017.013 
+0 0 20171017.014 
+0 0 20171017.015 
+0 0 20171024.013 
+0 0 20171024.014 
+0 0 20171025.005 
+0 0 20171025.006 
+0 0 20171026.005 
+0 0 20171026.006 
+0 0 20171026.007 
+0 0 20171101.012 
+0 0 20171101.013 
+0 0 20171101.019 
+0 0 20171102.007 
+0 0 20171102.009 
+0 0 20171107.015 
+0 0 20171107.016 
+0 0 20171108.007 
+0 0 20171108.008 
+0 0 20171108.009 
+0 0 20171108.010 
+0 0 20171108.011 
+0 0 20171108.013 
+0 0 20171109.005 
+0 0 20171109.006 
+0 0 20171114.012 
+0 0 20171114.013 
+0 0 20171114.014 
+0 0 20171114.016 
+0 0 20171114.022 
+0 0 20171114.023 
+0 0 20171115.006 
+0 0 20171121.007 
+0 0 20171121.008 
+0 0 20171121.009 
+0 0 20171121.010 
+0 0 20171122.006 
+0 0 20171122.007 
+0 0 20171122.008 
+0 0 20171123.008 
+0 0 20171123.009 
+0 0 20171123.010 
+0 0 20171123.011 
+0 0 20171123.012 
+0 0 20171123.013 
+0 0 20171129.008 
+0 0 20171129.009 
+0 0 20171205.006 
+0 0 20171205.007 
+0 0 20171206.013 
+0 0 20171207.006 
diff --git a/AEF51_coldframes_background_fails_real.txt b/AEF51_coldframes_background_fails_real.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8c5f7d706d6f56dfea82cf8f87e1b5b293c8a269
--- /dev/null
+++ b/AEF51_coldframes_background_fails_real.txt
@@ -0,0 +1,71 @@
+coldframes_fail_shots coldframe_value background_fail_shots ; numberofPrograms=928 ; numberoffails=69 
+20170926.009 0.00743059749162 20170926.010 
+20171207.034 0.41088691213 20170926.012 
+0 0 20170926.012 
+0 0 20170927.005 
+0 0 20171004.007 
+0 0 20171004.008 
+0 0 20171005.007 
+0 0 20171005.008 
+0 0 20171005.009 
+0 0 20171010.006 
+0 0 20171010.007 
+0 0 20171010.008 
+0 0 20171010.009 
+0 0 20171010.010 
+0 0 20171010.011 
+0 0 20171010.013 
+0 0 20171010.014 
+0 0 20171010.016 
+0 0 20171010.017 
+0 0 20171010.018 
+0 0 20171011.007 
+0 0 20171011.008 
+0 0 20171011.009 
+0 0 20171012.009 
+0 0 20171017.011 
+0 0 20171017.012 
+0 0 20171017.013 
+0 0 20171017.015 
+0 0 20171024.012 
+0 0 20171024.013 
+0 0 20171024.014 
+0 0 20171024.024 
+0 0 20171024.028 
+0 0 20171026.005 
+0 0 20171101.012 
+0 0 20171101.013 
+0 0 20171102.007 
+0 0 20171107.015 
+0 0 20171107.016 
+0 0 20171108.007 
+0 0 20171108.008 
+0 0 20171108.009 
+0 0 20171114.013 
+0 0 20171115.006 
+0 0 20171121.007 
+0 0 20171121.008 
+0 0 20171121.009 
+0 0 20171122.006 
+0 0 20171122.007 
+0 0 20171122.008 
+0 0 20171123.008 
+0 0 20171123.009 
+0 0 20171123.010 
+0 0 20171123.011 
+0 0 20171123.012 
+0 0 20171123.013 
+0 0 20171123.014 
+0 0 20171123.015 
+0 0 20171123.031 
+0 0 20171123.032 
+0 0 20171123.033 
+0 0 20171123.034 
+0 0 20171123.035 
+0 0 20171123.036 
+0 0 20171123.037 
+0 0 20171129.008 
+0 0 20171205.006 
+0 0 20171206.013 
+0 0 20171207.006 
+0 0 20171207.034 
diff --git a/IR_config_constants.py b/IR_config_constants.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2a4bfa72b9773382ce3c3590d5f774e8ac713eb
--- /dev/null
+++ b/IR_config_constants.py
@@ -0,0 +1,373 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Jun  5 09:26:06 2018
+
+@author: holn
+constants and config parameters
+"""
+#%% Paths
+archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/W7X/"
+calibpath="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Calibrations\\"
+IRCamColdframes_fittingpath=calibpath+"IRCamColdframes_fitting\\"
+IRCamRefImagespath = calibpath+'IRCamReferenceImages\\'
+
+#%% Dictionaries
+portcamdict={
+    'OP1.2a':{
+        'AEF10': 'IRCam_Caleo768kL_0901',
+        'AEF11': 'IRCam_Caleo768kL_0904',#is corrected from 0906, the cameras was swapped in the campaign in restect to the planned position
+        'AEF20': 'IRCam_Caleo768kL_0702',
+        'AEF21': 'IRCam_Caleo768kL_0906',#is corrected from 0904, the cameras was swapped in the campaign in restect to the planned position
+        'AEF30': 'IRCam_Caleo768kL_0907',
+        'AEF31': 'IRCam_Caleo768kL_0903',
+        'AEF40': 'IRCam_Caleo768kL_0701',
+        'AEF41': 'IRCam_Caleo768kL_0902',
+        'AEF50': 'Infratec_9312907',
+        'AEF51': 'IRCam_Caleo768kL_0905'}
+    	}
+     
+camlimdict={
+    'IRCam_Caleo768kL_0901':
+        {
+        1:(50,5000),
+        2:(50,3000),
+        3:(50,2050),
+        4:(50,1200),
+        5:(50,1100),
+        6:(50,850),
+        7:(50,800),
+        8:(50,600),
+        9:(50,550)
+         },
+    'IRCam_Caleo768kL_0902':
+        {
+        1:(50,5000),
+        2:(50,3000),
+        3:(50,2050),
+        4:(50,1200),
+        5:(50,1100),
+        6:(50,1000),
+        7:(50,800),
+        8:(50,800),
+        9:(50,650)
+         },
+    'IRCam_Caleo768kL_0903':
+        {
+        1:(50,5000),
+        2:(50,3000),
+        3:(50,2050),
+        4:(50,1200),
+        5:(50,1100),
+        6:(50,1000),
+        7:(50,900),
+        8:(50,800),
+        9:(50,700)
+         },
+    'IRCam_Caleo768kL_0904':
+        {
+        1:(50,5000),
+        2:(50,3000),
+        3:(50,2050),
+        4:(50,1200),
+        5:(50,1100),
+        6:(50,900),
+        7:(50,800),
+        8:(50,700),
+        9:(50,600)
+         },
+    'IRCam_Caleo768kL_0905':
+        {
+        1:(50,5000),
+        2:(50,3000),
+        3:(50,2050),
+        4:(50,1200),
+        5:(50,1100),
+        6:(50,900),
+        7:(50,800),
+        8:(50,700),
+        9:(50,600)
+         },
+    'IRCam_Caleo768kL_0906':
+        {
+        1:(50,5000),
+        2:(50,3000),
+        3:(50,2050),
+        4:(50,1200),
+        5:(50,1000),
+        6:(50,800),
+        7:(50,750),
+        8:(50,600),
+        9:(50,550)
+         },
+    'IRCam_Caleo768kL_0907':
+        {
+        1:(50,5000),
+        2:(50,3000),
+        3:(50,2050),
+        4:(50,1200),
+        5:(50,1100),
+        6:(50,900),
+        7:(50,800),
+        8:(50,700),
+        9:(50,600)
+         },
+    'IRCam_Caleo768kL_0701':        
+        {
+        1:(50,5000),
+        2:(50,3000),
+        3:(50,2050),
+        4:(50,1580),
+        5:(50,1100),
+        6:(50,900),
+        7:(50,800),
+        8:(50,750),
+        9:(50,700)
+         },
+    'IRCam_Caleo768kL_0702':
+        {
+        1:(50,5000),
+        2:(50,3000),
+        3:(50,2050),
+        4:(50,1580),
+        5:(50,1150),
+        6:(50,1000),
+        7:(50,800),
+        8:(50,750),
+        9:(50,700)
+         },
+     'Infratec_9312907':
+         {
+             0:{
+             25:(50,400),
+             50:(50,300),
+            100:(50,250),
+            200:(50,200),
+            400:(50,150),
+            800:(50,100)
+             },
+             1:{
+             25:(300,1200),
+             50:(200,800),
+            100:(150,650),
+            200:(100,500),
+            400:(50,400),
+            800:(50,300)
+             },
+            2:{
+             25:(200,1200),
+             50:(200,1200),
+            100:(200,1200),
+            200:(200,1200),
+            400:(200,1000),
+            800:(200,700)
+             },
+         }
+}
+
+valid_FOV_circle = {
+    10: [562, 364, 550],#, 600],
+    11: [502, 404, 520],#, 570]
+    20: [535, 374, 510],
+    21: [548, 404, 505],
+    30: [517, 389, 520],
+    31: [562, 364, 520],
+    40: [542, 394, 520],
+    41: [522, 394, 520],
+    50: [640, 454, 620],
+    51: [512, 379, 510]
+    }
+
+valid_background_rectangle = {
+    10: [500,0 ,800 ,100],#10: [300,0 ,800 ,200],
+    11: [50,200,170,290],#11: [200,100,700,200],
+    20: [100,100,275,135],#20: [100,100,400,200],
+    21: [300,0,700,75],#21: [150,0,800,200],
+    30: [200,0,800,100],#30: [200,0,800,200],#
+    31: [200,100,700,200],#
+    40: [550,0,690,140],#
+    41: [120,100,250,200],#
+    50: [390,940,880,1010],#
+    51: [350,0,670,120]#
+    }
+    
+TC_port={
+    10:[10],
+    11:[10],
+    20:[20],
+    21:[20],
+    30:[30],
+    31:[30],
+    40:[10,20,30],
+    41:[10,20,30],
+    50:[50],
+    51:[10,20,30]
+    }
+TC_channel={
+    10:[[66,67],["ActV1QSR10CT001","ActV1QSR10CT002"]], 
+    20:[[72,73],["ActV1QSR20CT001","ActV1QSR20CT002"]],
+    30:[[78,79],["ActV1QSR30CT001","ActV1QSR30CT002"]],
+    50:[[90,91,84,85],["ActV1QRT50CT001","ActV1QRT50CT002","ActV1QRT50CT003","ActV1QRT50CT004"]]
+    }
+    
+TC_database={
+    50: {
+        0: {'Y': -4220.44, 'Z': 966.369, 'Targetmodule': 'TM1h', 'active': True, 'X': 2947.882, 'Targetelement': 4}, 
+        1: {'Y': -4507.509, 'Z': 1000.004, 'Targetmodule': 'TM2h', 'active': True, 'X': 2673.037, 'Targetelement': 4}, 
+        2: {'Y': -4834.308, 'Z': 1006.452, 'Targetmodule': 'TM3h', 'active': True, 'X': 2339.889, 'Targetelement': 5}, 
+        3: {'Y': -5124.416, 'Z': 998.608, 'Targetmodule': 'TM4h', 'active': True, 'X': 1998.605, 'Targetelement': 5}, 
+        4: {'Y': -5862.386, 'Z': 749.666, 'Targetmodule': 'TM7h', 'active': True, 'X': 430.664, 'Targetelement': 4}, 
+        5: {'Y': -5971.865, 'Z': 676.355, 'Targetmodule': 'TM8h', 'active': True, 'X': -15.031, 'Targetelement': 6}, 
+        6: {'Y': -6065.916, 'Z': 542.714, 'Targetmodule': 'TM9h', 'active': True, 'X': -697.011, 'Targetelement': 6}, 
+        7: {'Y': -4068.477, 'Z': 644.281, 'Targetmodule': 'TM1v', 'active': True, 'X': 2704.554, 'Targetelement': 5}, 
+        8: {'Y': -4495.307, 'Z': 719.195, 'Targetmodule': 'TM2v', 'active': True, 'X': 2346.682, 'Targetelement': 6}, 
+        9: {'Y': -4841.007, 'Z': 771.012, 'Targetmodule': 'TM3v', 'active': True, 'X': 1981.551, 'Targetelement': 6}, 
+        10: {'Y': -5872.346, 'Z': 772.15, 'Targetmodule': 'DR_TM8h', 'active': True, 'X': 24.186, 'Targetelement': 0}, 
+        11: {'Y': -5910.584, 'Z': 726.881, 'Targetmodule': 'TMR_TM8h', 'active': True, 'X': 5.577, 'Targetelement': 0}, 
+        12: {'Y': -4703.059, 'Z': 1027.995, 'Targetmodule': 'TMR_TM3h', 'active': True, 'X': 2309.024, 'Targetelement': 0}, 
+        13: {'Y': -4945.311, 'Z': 1136.99, 'Targetmodule': 'DR_TM3h', 'active': True, 'X': 2464.537, 'Targetelement': 0}, 
+        14: {'Y': -4431.787, 'Z': 740.545, 'Targetmodule': 'TMR_TM2v', 'active': True, 'X': 2325.89, 'Targetelement': 0}, 
+        15: {'Y': -4322.24, 'Z': 714.53, 'Targetmodule': 'DR', 'active': True, 'X': 2374.085, 'Targetelement': 0}}, 
+    51: {0: {'Y': -5147.13, 'Z': -966.369, 'Targetmodule': 'TM1h', 'active': True, 'X': 95.826, 'Targetelement': 4}, 
+         1: {'Y': -5217.823, 'Z': -1000.004, 'Targetmodule': 'TM2h', 'active': True, 'X': 486.915, 'Targetelement': 4}, 
+        2: {'Y': -5286.39, 'Z': -1006.452, 'Targetmodule': 'TM3h', 'active': True, 'X': 948.525, 'Targetelement': 5}, 
+        3: {'Y': -5320.49, 'Z': -998.608, 'Targetmodule': 'TM4h', 'active': True, 'X': 1395.15, 'Targetelement': 5}, 
+        4: {'Y': -4995.908, 'Z': -749.666, 'Targetmodule': 'TM7h', 'active': True, 'X': 3097.409, 'Targetelement': 4}, 
+        5: {'Y': -4822.505, 'Z': -676.355, 'Targetmodule': 'TM8h', 'active': True, 'X': 3522.334, 'Targetelement': 6}, 
+        6: {'Y': -4497.737, 'Z': -542.714, 'Targetmodule': 'TM9h', 'active': True, 'X': 4129.35, 'Targetelement': 6}, 
+        7: {'Y': -4881.164, 'Z': -644.281, 'Targetmodule': 'TM1v', 'active': True, 'X': 203.361, 'Targetelement': 5}, 
+        8: {'Y': -5016.125, 'Z': -719.195, 'Targetmodule': 'TM2v', 'active': True, 'X': 743.77, 'Targetelement': 6}, 
+        9: {'Y': -5081.183, 'Z': -771.012, 'Targetmodule': 'TM3v', 'active': True, 'X': 1242.364, 'Targetelement': 6}, 
+        10: {'Y': -4779.849, 'Z': -772.15, 'Targetmodule': 'DR_TM8h', 'active': True, 'X': 3436.923, 'Targetelement': 0}, 
+        11: {'Y': -4860.039, 'Z': -726.881, 'Targetmodule': 'TMR_TM8h', 'active': True, 'X': 3494.011, 'Targetelement': 0}, 
+        12: {'Y': -5163.219, 'Z': -1027.995, 'Targetmodule': 'TMR_TM3h', 'active': True, 'X': 896.724, 'Targetelement': 0}, 
+        13: {'Y': -5447.878, 'Z': -1136.99, 'Targetmodule': 'DR_TM3h', 'active': True, 'X': 912.415, 'Targetelement': 0}, 
+        14: {'Y': -4893.659, 'Z': -740.545, 'Targetmodule': 'TMR_TM2v', 'active': True, 'X': 704.131, 'Targetelement': 0}, 
+        15: {'Y': -4949.37, 'Z': -714.53, 'Targetmodule': 'DR_TM2v', 'active': True, 'X': 638.444, 'Targetelement': 0}}, 
+    20: {0: {'Y': 5147.13, 'Z': 966.369, 'Targetmodule': 'TM1h', 'active': True, 'X': 95.826, 'Targetelement': 4}, 
+         1: {'Y': 5217.823, 'Z': 1000.004, 'Targetmodule': 'TM2h', 'active': True, 'X': 486.915, 'Targetelement': 4}, 
+        2: {'Y': 5286.39, 'Z': 1006.452, 'Targetmodule': 'TM3h', 'active': True, 'X': 948.525, 'Targetelement': 5}, 
+        3: {'Y': 5320.49, 'Z': 998.608, 'Targetmodule': 'TM4h', 'active': True, 'X': 1395.15, 'Targetelement': 5}, 
+        4: {'Y': 4995.908, 'Z': 749.666, 'Targetmodule': 'TM7h', 'active': True, 'X': 3097.409, 'Targetelement': 4}, 
+        5: {'Y': 4822.505, 'Z': 676.355, 'Targetmodule': 'TM8h', 'active': True, 'X': 3522.334, 'Targetelement': 6}, 
+        6: {'Y': 4497.737, 'Z': 542.714, 'Targetmodule': 'TM9h', 'active': True, 'X': 4129.35, 'Targetelement': 6}, 
+        7: {'Y': 4881.164, 'Z': 644.281, 'Targetmodule': 'TM1v', 'active': True, 'X': 203.361, 'Targetelement': 5}, 
+        8: {'Y': 5016.125, 'Z': 719.195, 'Targetmodule': 'TM2v', 'active': True, 'X': 743.77, 'Targetelement': 6}, 
+        9: {'Y': 5081.183, 'Z': 771.012, 'Targetmodule': 'TM3v', 'active': True, 'X': 1242.364, 'Targetelement': 6}, 
+        10: {'Y': 4765.044, 'Z': 772.15, 'Targetmodule': 'DR_TM8h', 'active': True, 'X': 3432.112, 'Targetelement': 0}, 
+        11: {'Y': 4785.041, 'Z': 726.881, 'Targetmodule': 'TMR_TM8h', 'active': True, 'X': 3469.642, 'Targetelement': 0}, 
+        12: {'Y': 5162.065, 'Z': 1027.995, 'Targetmodule': 'TMR_TM3h', 'active': True, 'X': 896.349, 'Targetelement': 0}, 
+        13: {'Y': 5449.459, 'Z': 1136.99, 'Targetmodule': 'DR_TM3h', 'active': True, 'X': 912.928, 'Targetelement': 0}, 
+        14: {'Y': 4952.514, 'Z': 740.545, 'Targetmodule': 'TMR_TM2v', 'active': True, 'X': 723.255, 'Targetelement': 0}, 
+        15: {'Y': 4892.218, 'Z': 714.53, 'Targetmodule': 'DR_TM2v', 'active': True, 'X': 619.874, 'Targetelement': 0}}, 
+    21: {0: {'Y': 4220.44, 'Z': -966.369, 'Targetmodule': 'TM1h', 'active': True, 'X': 2947.882, 'Targetelement': 4}, 
+         1: {'Y': 4507.509, 'Z': -1000.004, 'Targetmodule': 'TM2h', 'active': False, 'X': 2673.037, 'Targetelement': 4}, 
+        2: {'Y': 4834.308, 'Z': -1006.452, 'Targetmodule': 'TM3h', 'active': True, 'X': 2339.889, 'Targetelement': 5}, 
+        3: {'Y': 5124.416, 'Z': -998.608, 'Targetmodule': 'TM4h', 'active': True, 'X': 1998.605, 'Targetelement': 5}, 
+        4: {'Y': 5862.386, 'Z': -749.666, 'Targetmodule': 'TM7h', 'active': True, 'X': 430.664, 'Targetelement': 4}, 
+        5: {'Y': 5971.865, 'Z': -676.355, 'Targetmodule': 'TM8h', 'active': True, 'X': -15.031, 'Targetelement': 6}, 
+        6: {'Y': 6065.916, 'Z': -542.714, 'Targetmodule': 'TM9h', 'active': True, 'X': -697.011, 'Targetelement': 6}, 
+        7: {'Y': 4068.477, 'Z': -644.281, 'Targetmodule': 'TM1v', 'active': True, 'X': 2704.554, 'Targetelement': 5}, 
+        8: {'Y': 4495.307, 'Z': -719.195, 'Targetmodule': 'TM2v', 'active': True, 'X': 2346.682, 'Targetelement': 6}, 
+        9: {'Y': 4841.007, 'Z': -771.012, 'Targetmodule': 'TM3v', 'active': True, 'X': 1981.551, 'Targetelement': 6}, 
+        10: {'Y': 5887.151, 'Z': -772.15, 'Targetmodule': 'DR_TM8h', 'active': True, 'X': 28.996, 'Targetelement': 0}, 
+        11: {'Y': 5985.582, 'Z': -726.881, 'Targetmodule': 'TMR_TM8h', 'active': True, 'X': 29.945, 'Targetelement': 0}, 
+        12: {'Y': 4704.213, 'Z': -1027.995, 'Targetmodule': 'TMR_TM3h', 'active': True, 'X': 2309.399, 'Targetelement': 0}, 
+        13: {'Y': 4943.73, 'Z': -1136.99, 'Targetmodule': 'DR_TM3h', 'active': True, 'X': 2464.024, 'Targetelement': 0}, 
+        14: {'Y': 4372.932, 'Z': -740.545, 'Targetmodule': 'TMR_TM2v', 'active': True, 'X': 2306.766, 'Targetelement': 0}, 
+        15: {'Y': 4379.393, 'Z': -714.53, 'Targetmodule': 'DR_TM2v', 'active': True, 'X': 2392.655, 'Targetelement': 0}}, 
+    40: {0: {'Y': -4107.791, 'Z': 966.369, 'Targetmodule': 'TM1h', 'active': True, 'X': -3102.931, 'Targetelement': 4}, 
+         1: {'Y': -3935.106, 'Z': 1000.004, 'Targetmodule': 'TM2h', 'active': True, 'X': -3460.882, 'Targetelement': 4}, 
+        2: {'Y': -3719.25, 'Z': 1006.452, 'Targetmodule': 'TM3h', 'active': True, 'X': -3874.635, 'Targetelement': 5}, 
+        3: {'Y': -3484.318, 'Z': 998.608, 'Targetmodule': 'TM4h', 'active': True, 'X': -4256.006, 'Targetelement': 5}, 
+        4: {'Y': -2221.163, 'Z': 749.666, 'Targetmodule': 'TM7h', 'active': True, 'X': -5442.378, 'Targetelement': 4}, 
+        5: {'Y': -1831.112, 'Z': 676.355, 'Targetmodule': 'TM8h', 'active': True, 'X': -5684.226, 'Targetelement': 6}, 
+        6: {'Y': -1211.574, 'Z': 542.714, 'Targetmodule': 'TM9h', 'active': True, 'X': -5984.417, 'Targetelement': 6}, 
+        7: {'Y': -3829.412, 'Z': 644.281, 'Targetmodule': 'TM1v', 'active': True, 'X': -3033.599, 'Targetelement': 5}, 
+        8: {'Y': -3620.954, 'Z': 719.195, 'Targetmodule': 'TM2v', 'active': True, 'X': -3550.127, 'Targetelement': 6}, 
+        9: {'Y': -3380.521, 'Z': 771.012, 'Targetmodule': 'TM3v', 'active': True, 'X': -3991.738, 'Targetelement': 6}, 
+        10: {'Y': -1837.657, 'Z': 772.15, 'Targetmodule': 'DR_TM8h', 'active': True, 'X': -5577.46, 'Targetelement': 0}, 
+        11: {'Y': -1831.775, 'Z': 726.881, 'Targetmodule': 'TMR_TM8h', 'active': True, 'X': -5619.576, 'Targetelement': 0}, 
+        12: {'Y': -3649.338, 'Z': 1027.995, 'Targetmodule': 'TMR_TM3h', 'active': True, 'X': -3759.348, 'Targetelement': 0}, 
+        13: {'Y': -3872.099, 'Z': 1136.99, 'Targetmodule': 'DR_TM3h', 'active': True, 'X': -3941.686, 'Targetelement': 0}, 
+        14: {'Y': -3581.55, 'Z': 740.545, 'Targetmodule': 'TMR_TM2v', 'active': True, 'X': -3496.14, 'Targetelement': 0}, 
+        15: {'Y': -3593.534, 'Z': 714.53, 'Targetmodule': 'DR_TM2v', 'active': True, 'X': -3377.062, 'Targetelement': 0}}, 
+    41: {0: {'Y': -1681.686, 'Z': -966.369, 'Targetmodule': 'TM1h', 'active': True, 'X': -4865.6, 'Targetelement': 4}, 
+         1: {'Y': -2075.48, 'Z': -1000.004, 'Targetmodule': 'TM2h', 'active': True, 'X': -4811.98, 'Targetelement': 4}, 
+        2: {'Y': -2535.686, 'Z': -1006.452, 'Targetmodule': 'TM3h', 'active': True, 'X': -4734.545, 'Targetelement': 5}, 
+        3: {'Y': -2970.989, 'Z': -998.608, 'Targetmodule': 'TM4h', 'active': True, 'X': -4628.962, 'Targetelement': 5}, 
+        4: {'Y': -4489.632, 'Z': -749.666, 'Targetmodule': 'TM7h', 'active': True, 'X': -3794.239, 'Targetelement': 4}, 
+        5: {'Y': -4840.175, 'Z': -676.355, 'Targetmodule': 'TM8h', 'active': True, 'X': -3498.014, 'Targetelement': 6}, 
+        6: {'Y': -5317.122, 'Z': -542.714, 'Targetmodule': 'TM9h', 'active': True, 'X': -3001.562, 'Targetelement': 6}, 
+        7: {'Y': -1701.77, 'Z': -644.281, 'Targetmodule': 'TM1v', 'active': True, 'X': -4579.421, 'Targetelement': 5}, 
+        8: {'Y': -2257.435, 'Z': -719.195, 'Targetmodule': 'TM2v', 'active': True, 'X': -4540.781, 'Targetelement': 6}, 
+        9: {'Y': -2751.73, 'Z': -771.012, 'Targetmodule': 'TM3v', 'active': True, 'X': -4448.581, 'Targetelement': 6}, 
+        10: {'Y': -4745.762, 'Z': -772.15, 'Targetmodule': 'DR_TM8h', 'active': True, 'X': -3483.839, 'Targetelement': 0}, 
+        11: {'Y': -4824.836, 'Z': -726.881, 'Targetmodule': 'TMR_TM8h', 'active': True, 'X': -3542.463, 'Targetelement': 0}, 
+        12: {'Y': -2448.358, 'Z': -1027.995, 'Targetmodule': 'TMR_TM3h', 'active': True, 'X': -4633.41, 'Targetelement': 0}, 
+        13: {'Y': -2551.245, 'Z': -1136.99, 'Targetmodule': 'DR_TM3h', 'active': True, 'X': -4899.289, 'Targetelement': 0}, 
+        14: {'Y': -2181.893, 'Z': -740.545, 'Targetmodule': 'TMR_TM2v', 'active': True, 'X': -4436.558, 'Targetelement': 0}, 
+        15: {'Y': -2136.636, 'Z': -714.53, 'Targetmodule': 'DR_TM2v', 'active': True, 'X': -4509.841, 'Targetelement': 0}}, 
+    10: {0: {'Y': 1499.415, 'Z': 966.369, 'Targetmodule': 'TM1h', 'active': True, 'X': 4924.823, 'Targetelement': 4}, 
+         1: {'Y': 1149.312, 'Z': 1000.004, 'Targetmodule': 'TM2h', 'active': True, 'X': 5112.91, 'Targetelement': 4}, 
+        2: {'Y': 731.483, 'Z': 1006.452, 'Targetmodule': 'TM3h', 'active': True, 'X': 5320.766, 'Targetelement': 5}, 
+        3: {'Y': 317.255, 'Z': 998.608, 'Targetmodule': 'TM4h', 'active': True, 'X': 5491.212, 'Targetelement': 5}, 
+        4: {'Y': -1401.991, 'Z': 749.666, 'Targetmodule': 'TM7h', 'active': True, 'X': 5708.543, 'Targetelement': 4}, 
+        5: {'Y': -1859.703, 'Z': 676.355, 'Targetmodule': 'TM8h', 'active': True, 'X': 5674.936, 'Targetelement': 6}, 
+        6: {'Y': -2537.368, 'Z': 542.714, 'Targetmodule': 'TM9h', 'active': True, 'X': 5553.641, 'Targetelement': 6}, 
+        7: {'Y': 1314.955, 'Z': 644.281, 'Targetmodule': 'TM1v', 'active': True, 'X': 4705.105, 'Targetelement': 5}, 
+        8: {'Y': 842.701, 'Z': 719.195, 'Targetmodule': 'TM2v', 'active': True, 'X': 5000.456, 'Targetelement': 6}, 
+        9: {'Y': 388.614, 'Z': 771.012, 'Targetmodule': 'TM3v', 'active': True, 'X': 5216.404, 'Targetelement': 6}, 
+        10: {'Y': -1791.653, 'Z': 772.15, 'Targetmodule': 'DR_TM8h', 'active': True, 'X': 5592.407, 'Targetelement': 0}, 
+        11: {'Y': -1821.167, 'Z': 726.881, 'Targetmodule': 'TMR_TM8h', 'active': True, 'X': 5623.023, 'Targetelement': 0}, 
+        12: {'Y': 742.687, 'Z': 1027.995, 'Targetmodule': 'TMR_TM3h', 'active': True, 'X': 5186.403, 'Targetelement': 0}, 
+        13: {'Y': 815.729, 'Z': 1136.99, 'Targetmodule': 'DR_TM3h', 'active': True, 'X': 5464.854, 'Targetelement': 0}, 
+        14: {'Y': 842.555, 'Z': 740.545, 'Targetmodule': 'TMR_TM2v', 'active': True, 'X': 4933.619, 'Targetelement': 0}, 
+        15: {'Y': 922.243, 'Z': 714.53, 'Targetmodule': 'DR_TM2v', 'active': True, 'X': 4844.327, 'Targetelement': 0}}, 
+    11: {0: {'Y': -1499.415, 'Z': -966.369, 'Targetmodule': 'TM1h', 'active': True, 'X': 4924.823, 'Targetelement': 4}, 
+         1: {'Y': -1149.312, 'Z': -1000.004, 'Targetmodule': 'TM2h', 'active': True, 'X': 5112.91, 'Targetelement': 4}, 
+        2: {'Y': -731.483, 'Z': -1006.452, 'Targetmodule': 'TM3h', 'active': True, 'X': 5320.766, 'Targetelement': 5}, 
+        3: {'Y': -317.255, 'Z': -998.608, 'Targetmodule': 'TM4h', 'active': True, 'X': 5491.212, 'Targetelement': 5}, 
+        4: {'Y': 1401.991, 'Z': -749.666, 'Targetmodule': 'TM7h', 'active': True, 'X': 5708.543, 'Targetelement': 4}, 
+        5: {'Y': 1859.703, 'Z': -676.355, 'Targetmodule': 'TM8h', 'active': True, 'X': 5674.936, 'Targetelement': 6}, 
+        6: {'Y': 2537.368, 'Z': -542.714, 'Targetmodule': 'TM9h', 'active': True, 'X': 5553.641, 'Targetelement': 6}, 
+        7: {'Y': -1314.955, 'Z': -644.281, 'Targetmodule': 'TM1v', 'active': True, 'X': 4705.105, 'Targetelement': 5}, 
+        8: {'Y': -842.701, 'Z': -719.195, 'Targetmodule': 'TM2v', 'active': True, 'X': 5000.456, 'Targetelement': 6}, 
+        9: {'Y': -388.614, 'Z': -771.012, 'Targetmodule': 'TM3v', 'active': True, 'X': 5216.404, 'Targetelement': 6}, 
+        10: {'Y': 1791.653, 'Z': -772.15, 'Targetmodule': 'DR_TM8h', 'active': True, 'X': 5607.974, 'Targetelement': 0}, 
+        11: {'Y': 1821.167, 'Z': -726.881, 'Targetmodule': 'TMR_TM8h', 'active': True, 'X': 5701.88, 'Targetelement': 0}, 
+        12: {'Y': -742.687, 'Z': -1027.995, 'Targetmodule': 'TMR_TM3h', 'active': True, 'X': 5187.616, 'Targetelement': 0}, 
+        13: {'Y': -815.729, 'Z': -1136.99, 'Targetmodule': 'DR_TM3h', 'active': True, 'X': 5463.192, 'Targetelement': 0}, 
+        14: {'Y': -842.555, 'Z': -740.545, 'Targetmodule': 'TMR_TM2v', 'active': True, 'X': 4871.735, 'Targetelement': 0}, 
+        15: {'Y': -922.243, 'Z': -714.53, 'Targetmodule': 'DR_TM2v', 'active': True, 'X': 4904.421, 'Targetelement': 0}}, 
+    30: {0: {'Y': 1681.686, 'Z': 966.369, 'Targetmodule': 'TM1h', 'active': True, 'X': -4865.6, 'Targetelement': 4}, 
+         1: {'Y': 2075.48, 'Z': 1000.004, 'Targetmodule': 'TM2h', 'active': True, 'X': -4811.98, 'Targetelement': 4}, 
+        2: {'Y': 2535.686, 'Z': 1006.452, 'Targetmodule': 'TM3h', 'active': True, 'X': -4734.545, 'Targetelement': 5}, 
+        3: {'Y': 2970.989, 'Z': 998.608, 'Targetmodule': 'TM4h', 'active': True, 'X': -4628.962, 'Targetelement': 5}, 
+        4: {'Y': 4489.632, 'Z': 749.666, 'Targetmodule': 'TM7h', 'active': True, 'X': -3794.239, 'Targetelement': 4}, 
+        5: {'Y': 4840.175, 'Z': 676.355, 'Targetmodule': 'TM8h', 'active': True, 'X': -3498.014, 'Targetelement': 6}, 
+        6: {'Y': 5317.122, 'Z': 542.714, 'Targetmodule': 'TM9h', 'active': True, 'X': -3001.562, 'Targetelement': 6}, 
+        7: {'Y': 1701.77, 'Z': 644.281, 'Targetmodule': 'TM1v', 'active': True, 'X': -4579.421, 'Targetelement': 5}, 
+        8: {'Y': 2257.435, 'Z': 719.195, 'Targetmodule': 'TM2v', 'active': True, 'X': -4540.781, 'Targetelement': 6}, 
+        9: {'Y': 2751.73, 'Z': 771.012, 'Targetmodule': 'TM3v', 'active': True, 'X': -4448.581, 'Targetelement': 6}, 
+        10: {'Y': 4736.612, 'Z': 772.15, 'Targetmodule': 'DR_TM8h', 'active': True, 'X': -3471.245, 'Targetelement': 0}, 
+        11: {'Y': 4778.485, 'Z': 726.881, 'Targetmodule': 'TMR_TM8h', 'active': True, 'X': -3478.666, 'Targetelement': 0}, 
+        12: {'Y': 2447.645, 'Z': 1027.995, 'Targetmodule': 'TMR_TM3h', 'active': True, 'X': -4632.429, 'Targetelement': 0}, 
+        13: {'Y': 2552.222, 'Z': 1136.99, 'Targetmodule': 'DR_TM3h', 'active': True, 'X': -4900.633, 'Targetelement': 0}, 
+        14: {'Y': 2218.267, 'Z': 740.545, 'Targetmodule': 'TMR_TM2v', 'active': True, 'X': -4486.623, 'Targetelement': 0}, 
+        15: {'Y': 2101.314, 'Z': 714.53, 'Targetmodule': 'DR_TM2v', 'active': True, 'X': -4461.224, 'Targetelement': 0}}, 
+    31: {0: {'Y': 4107.791, 'Z': -966.369, 'Targetmodule': 'TM1h', 'active': True, 'X': -3102.931, 'Targetelement': 4}, 
+         1: {'Y': 3935.106, 'Z': -1000.004, 'Targetmodule': 'TM2h', 'active': True, 'X': -3460.882, 'Targetelement': 4}, 
+        2: {'Y': 3719.25, 'Z': -1006.452, 'Targetmodule': 'TM3h', 'active': True, 'X': -3874.635, 'Targetelement': 5}, 
+        3: {'Y': 3484.318, 'Z': -998.608, 'Targetmodule': 'TM4h', 'active': True, 'X': -4256.006, 'Targetelement': 5}, 
+        4: {'Y': 2221.163, 'Z': -749.666, 'Targetmodule': 'TM7h', 'active': True, 'X': -5442.378, 'Targetelement': 4}, 
+        5: {'Y': 1831.112, 'Z': -676.355, 'Targetmodule': 'TM8h', 'active': True, 'X': -5684.226, 'Targetelement': 6}, 
+        6: {'Y': 1211.574, 'Z': -542.714, 'Targetmodule': 'TM9h', 'active': True, 'X': -5984.417, 'Targetelement': 5}, 
+        7: {'Y': 3829.412, 'Z': -644.281, 'Targetmodule': 'TM1v', 'active': True, 'X': -3033.599, 'Targetelement': 5}, 
+        8: {'Y': 3620.954, 'Z': -719.195, 'Targetmodule': 'TM2v', 'active': True, 'X': -3550.127, 'Targetelement': 6}, 
+        9: {'Y': 3380.521, 'Z': -771.012, 'Targetmodule': 'TM3v', 'active': True, 'X': -3991.738, 'Targetelement': 6}, 
+        10: {'Y': 1846.807, 'Z': -772.15, 'Targetmodule': 'DR_TM8h', 'active': True, 'X': -5590.053, 'Targetelement': 0}, 
+        11: {'Y': 1878.126, 'Z': -726.881, 'Targetmodule': 'TMR_TM8h', 'active': True, 'X': -5683.373, 'Targetelement': 0}, 
+        12: {'Y': 3650.051, 'Z': -1027.995, 'Targetmodule': 'TMR_TM3h', 'active': True, 'X': -3760.329, 'Targetelement': 0}, 
+        13: {'Y': 3871.122, 'Z': -1136.99, 'Targetmodule': 'DR_TM3h', 'active': True, 'X': -3940.342, 'Targetelement': 0}, 
+        14: {'Y': 3545.175, 'Z': -740.545, 'Targetmodule': 'TMR_TM2v', 'active': True, 'X': -3446.075, 'Targetelement': 0}, 
+        15: {'Y': 3628.857, 'Z': -714.53, 'Targetmodule': 'DR_TM2v', 'active': True, 'X': -3425.679, 'Targetelement': 0}}}
\ No newline at end of file
diff --git a/IR_image_tools.py b/IR_image_tools.py
new file mode 100644
index 0000000000000000000000000000000000000000..fcdd289415ed2d9a322e574ae3d4668aef74564a
--- /dev/null
+++ b/IR_image_tools.py
@@ -0,0 +1,314 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Wed May  9 14:56:32 2018
+
+@author: Holger Niemann, Peter Drewelow, Yu Gao
+
+mainly to clean up the downloadversionIRdata code
+Tools for:
+    checking IR images, 
+    calculate gain and offset again from 
+    check backgroundframes
+    check coldframes
+    ...
+"""
+import numpy as np
+import matplotlib.pyplot as plt
+from IR_config_constants import portcamdict,IRCamRefImagespath
+import h5py
+from os.path import join, basename
+import glob
+
+
+
+def bestimmtheitsmaß_general(data,fit):
+    R=0
+    if len(fit)==len(data):
+        mittel=np.sum(data)/len(data)
+        qam=quad_abweich_mittel(fit,mittel)
+        R=qam/(qam+quad_abweich(data,fit))
+    else:
+        print("Arrays must have same dimensions")
+    return R
+    
+def quad_abweich_mittel(data,mittel):
+    R=0
+    for i in data:
+        R=R+(i-mittel)**2
+    return R
+    
+def quad_abweich(data,fit):
+    R=0
+    if len(fit)==len(data):
+        for i in range(len(data)):
+            R=R+(data[i]-fit[i])**2
+    else:
+        print("Arrays must have same dimensions")
+    return R    
+        
+def check_coldframe(coldframe,references=None,threshold=0.5,plot_it=False):
+    '''
+    return true/false and the quality factor
+    '''
+    shapi=np.shape(coldframe)
+    ##function  (np.arange(0,768)-384)**(2)/900-50
+    datasets=[]
+    for i in [int(shapi[1]//4),int(shapi[1]//2),int(shapi[1]//4*3)]:
+        dataline=coldframe[0:shapi[0],i]    
+        datasets.append(dataline-np.mean(dataline))
+    if references==None:
+        references=[]
+        for dat in datasets:        
+            mini=np.mean(dat[shapi[0]/2-50:shapi[0]/2+50])
+            a=(np.mean(dat[0:50])-mini)/(int(shapi[0]/2))**2
+            reference=a*(np.arange(0,shapi[0])-int(shapi[0]/2))**(2)+mini
+            references.append(reference)
+    bestimmtheit=[]
+    if plot_it:
+        plt.figure()
+        plt.imshow(coldframe,vmin=np.mean(coldframe)-500,vmax=np.mean(coldframe)+500)
+        plt.figure()
+    for i_dat in range(len(datasets)):
+        dat=datasets[i_dat]
+        reference=references[i_dat]
+        bestimmtheit.append(bestimmtheitsmaß_general(dat,reference))
+        if plot_it:            
+            plt.plot(dat,label='data')
+            plt.plot(reference,label='reference')
+#            print(int(shapi[0]/2),1*(np.max(datasets[-1])-mini),mini)
+            plt.legend()
+    if np.mean(bestimmtheit)>threshold:
+        return True,bestimmtheit
+    else:
+        return False,bestimmtheit
+
+def check_coldframe_by_refframe(coldframe,reference_frame,threshold=0.8,plot_it=False):
+    references=[]
+    shapi=np.shape(reference_frame)
+    for i in [int(shapi[1]//5),int(shapi[1]//2),int(shapi[1]//4*3)]:
+        dataline=reference_frame[0:shapi[0],i]    
+        references.append(dataline-np.mean(dataline))
+    return check_coldframe(coldframe,references,threshold,plot_it)
+        
+def check_backgroundframe(backgroundframe,threshold=50):
+    '''
+    return true or false
+    '''
+    shapi=np.shape(backgroundframe)
+    valid=True
+    dataset=[]
+    for i in [int(shapi[1]//4),int(shapi[1]//2),int(shapi[1]//4*3)]:
+        referenceline=backgroundframe[0:shapi[0],i]    
+        meanref=referenceline-np.mean(referenceline)
+        dataset.append(np.max(meanref)-np.min(meanref))
+    if np.mean(dataset)<threshold:
+        valid=False    
+    return valid,np.mean(dataset)
+    
+def find_outlier_pixels(frame,tolerance=3,worry_about_edges=True,plot_it=False):
+    # This function finds the bad pixels in a 2D dataset. 
+    # Tolerance is the number of standard deviations used for cutoff.
+    frame = np.array(frame)#, dtype=int)
+    from scipy.ndimage import median_filter
+    blurred = median_filter(frame, size=9)
+    difference = frame - blurred
+    threshold = tolerance*np.std(difference)
+    mean = np.mean(difference)
+    if plot_it:
+        
+        
+        fig = plt.figure()
+        fig.suptitle('find_outlier_pixels: histogram')
+        plt.hist(difference.ravel(),50,log=True,histtype='stepfilled')
+        plt.axvline(mean, linewidth=2, color='k',label='mean')
+        x1 = mean - np.std(difference)
+        x2 = mean + np.std(difference)
+        plt.axvspan(x1,x2, linewidth=2, facecolor='g',alpha=0.1,label='standard deviation')
+        x1 = mean - tolerance*np.std(difference)
+        x2 = mean + tolerance*np.std(difference)
+        plt.axvspan(x1,x2, linewidth=2, facecolor='r',alpha=0.1,label='threshold for bad pixel')
+        plt.legend()
+        plt.show()
+        
+    #find the hot pixels
+    bad_pixels = np.transpose(np.nonzero((np.abs(difference)>threshold)) )
+    bad_pixels = (bad_pixels).tolist()
+    bad_pixels = [tuple(l) for l in bad_pixels]
+    
+    if plot_it:
+        plt.figure()
+        plt.imshow(frame)
+        for i in range(len(bad_pixels)):
+            plt.scatter(bad_pixels[i][1],bad_pixels[i][0],c='None')
+        plt.show()
+   
+    return bad_pixels
+
+def correct_images(images,badpixels):
+    if type(badpixels)!=int:
+        for i in range(len(images)):
+            images[i]=(restore_pixels(images[i],np.invert(badpixels==1))).astype(np.float32)
+        print("done")
+    return images
+
+def restore_pixels(frame, bad_pixel):# code from Peter from JET
+    # make sure bad pixel are provided as mask and list  
+    if type(bad_pixel) is list:
+        blist = bad_pixel
+        bmask = np.ones(frame.shape,dtype=bool)
+        for pix in bad_pixel:
+            bmask[pix] = False
+        bmask = np.invert(bmask)
+    else:
+        bmask = np.invert(bad_pixel)
+        x,y = np.where(bmask)
+        blist = list(zip(x,y))
+#    print('restore_pixels() working on: ',blist)
+    # prepare internal result frames
+    resframe = frame.astype(float)
+    # -----------------------------------
+    # restore by list
+    #
+    height = np.shape(frame)[0]
+    width  = np.shape(frame)[1]
+    for pos in blist:
+        # assume four neighbouring, non-bad pixels exist
+        n_neighbours = 4
+        # -- top neighbour --
+        if pos[0] == 0:
+            # out of bounds
+            top = 0
+            n_neighbours -= 1
+        else:
+            i_t = pos[0] - 1
+            while (bmask[i_t,pos[1]] & (i_t >= 0)):
+                i_t -= 1
+            if i_t >= 0:
+                top = frame[i_t,pos[1]]
+            else:
+                # out of bounds
+                top = 0
+                n_neighbours -= 1
+            #print('top: ',i_t, top)    
+        # -- bottom neighbour --
+        if pos[0] == height-1:
+            # out of bounds
+            bottom = 0
+            n_neighbours -= 1
+        else:
+            i_b = pos[0] + 1
+            while (bmask[i_b,pos[1]] & (i_b <= height-2)):
+                i_b += 1
+            if i_b >= 0:
+                bottom = frame[i_b,pos[1]]
+            else:
+                # out of bounds
+                bottom = 0
+                n_neighbours -= 1
+            #print('bottom: ',i_b, bottom)
+        # -- left neighbour --
+        if pos[1] == 0:
+            # out of bounds
+            left = 0
+            n_neighbours -= 1
+        else:
+            i_l = pos[1] - 1
+            while (bmask[pos[0],i_l] & (i_l >= 0)):
+                i_l -= 1
+            if i_l >= 0:
+                left = frame[pos[0],i_l]
+            else:
+                # out of bounds
+                left = 0
+                n_neighbours -= 1
+            #print('left: ',i_l, left)    
+        # -- right neighbour --
+        if pos[1] == width-1:
+            # out of bounds
+            right = 0
+            n_neighbours -= 1
+        else:
+            i_r = pos[1] + 1
+            while (bmask[pos[0],i_r] & (i_r <= width-2)):
+                i_r += 1
+            if i_r >= 0:
+                right = frame[pos[0],i_r]
+            else:
+                # out of bounds
+                right = 0
+                n_neighbours -= 1
+            #print('right: ',i_r, right)                
+        # averaging
+        if n_neighbours > 0:
+            #print('original value: ',frame[pos[0],pos[1]])
+            resframe[pos[0],pos[1]] = (top + bottom + left + right)/n_neighbours
+            #print('average of ',n_neighbours,' neighbours: ',frame1[pos[0],pos[1]])
+        else:
+            print('ERROR: no adjacent pixel found!')
+    return resframe
+
+def generate_new_hot_image(cold,reference_cold,reference_hot):
+    if cold==None or reference_cold==None or reference_hot==None:
+        raise Exception("Cannot Calculate new Hot image, if images are missing!")
+    else:
+        return reference_hot+(cold-reference_cold)
+    
+def calculate_gain_offset_image_pix(cold_image,hot_image=None,reference_cold=None,reference_hot=None,bose=1):    
+    if hot_image==None:
+        hot_image=generate_new_hot_image(cold_image,reference_cold,reference_hot)
+    if bose>0:
+        print("calculate gain and offset")        
+    Sh_ref =  hot_image[ ( np.int( np.shape(hot_image)[0]   /2  )  ) ][np.int( (np.shape(hot_image)[1]   /2  ) ) ]          
+    Sc_ref =  cold_image[ ( np.int(  (np.shape(cold_image)[0])  /2 )  ) ][( np.int(  (np.shape(cold_image)[1])  /2 ) ) ]  
+    Gain_rel =  ( Sh_ref  - Sc_ref ) / ( hot_image  - cold_image)    
+    Off_h_rel = Sh_ref -   hot_image*Gain_rel
+    Off_c_rel = Sc_ref -   cold_image*Gain_rel    
+    Offset_rel  = ( Off_h_rel + Off_c_rel ) /2
+    return Gain_rel,Offset_rel
+
+def calculate_gain_offset_image(cold_image,hot_image=None,reference_cold=None,reference_hot=None,bose=1):    
+    if hot_image==None:
+        hot_image=generate_new_hot_image(cold_image,reference_cold,reference_hot)
+    if bose>0:
+        print("calculate gain and offset")  
+    
+#    Sh_ref =  hot_image[ ( np.int( np.shape(hot_image)[0]   /2  )  ) ][np.int( (np.shape(hot_image)[1]   /2  ) ) ]          
+#    Sc_ref =  cold_image[ ( np.int(  (np.shape(cold_image)[0])  /2 )  ) ][( np.int(  (np.shape(cold_image)[1])  /2 ) ) ]  
+    print(hot_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])
+    print(cold_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])
+    Sh_ref =  np.mean( hot_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])    
+    Sc_ref =  np.mean(cold_image[( np.int( np.shape(cold_image)[0]/2) )-2: (np.int( np.shape(cold_image)[0]/2))+3,np.int((np.shape(cold_image)[1]/2))-2:np.int((np.shape(cold_image)[1]/2))+3 ])    
+    Gain_rel =  ( Sh_ref  - Sc_ref ) / ( hot_image  - cold_image)    
+    Off_h_rel = Sh_ref -   hot_image*Gain_rel
+    Off_c_rel = Sc_ref -   cold_image*Gain_rel    
+    Offset_rel  = ( Off_h_rel + Off_c_rel ) /2
+    return Gain_rel,Offset_rel    
+
+    
+#%% functions from Yu Gao
+""" functions by Yu Gao"""
+    
+def load_ref_images(port, exposuretime):
+  '''
+  load the reference cold and hot frame during calibration from local files.
+  @port: e.g. 'AEF10'
+  @exposuretime: int number.
+  '''
+  cameraname = portcamdict['OP1.2a'][port]
+  foldername = cameraname.split('_')[0] + '_' + cameraname.split('_')[2]
+  scanpath = join(IRCamRefImagespath, foldername)
+  coldref, hotref = [], []
+  for filename in glob.iglob(scanpath + '\*' + str(int(exposuretime)) + 'us.h5', recursive=True):
+    if 'hot' in filename:
+      print (filename)
+      with h5py.File(filename, 'r') as h5in:
+        hotref = h5in[basename(filename)].value
+    elif 'cold' in filename:
+      print (filename)
+      with h5py.File(filename, 'r') as h5in:
+        coldref = h5in[basename(filename)].value
+  return coldref, hotref
+
+def reconstruct_coldframe (exposuretime, sT, a, bnew, coldref):
+  cirebuild = a * sT + bnew * exposuretime + coldref
+  return cirebuild
\ No newline at end of file
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..1072fc76dd139a40ea10043210948592a858034b
--- /dev/null
+++ b/__init__.py
@@ -0,0 +1,3 @@
+from . import IR_image_tools
+from . import downloadversionIRdata
+from . import IR_config_constants
\ No newline at end of file
diff --git a/downloadversionIRdata.py b/downloadversionIRdata.py
index caf3503970aa4977127e6df2a0b4b6cc0b9df32c..d02102d540a94d90b368353535ab98de73b36e2b 100644
--- a/downloadversionIRdata.py
+++ b/downloadversionIRdata.py
@@ -6,6 +6,8 @@ Created on Wed Oct 25 15:51:46 2017
 """
 
 import numpy as np
+import IR_image_tools as IR_tools
+from IR_config_constants import archivepath,portcamdict,camlimdict,valid_FOV_circle,valid_background_rectangle,TC_port,TC_channel,IRCamColdframes_fittingpath
 try:
     import W7Xrest.read_restdb as AKF_1
 except:
@@ -15,33 +17,16 @@ import urllib
 import json
 from PIL import Image
 from io import BytesIO
+from os.path import join
 import matplotlib.pyplot as plt
 try:        
     import multiprocessing    
-    try:
-        import holn.archivedb as AKF_2
-    except:
-        import archivedb as AKF_2
+    import archivedb as AKF_2
     fastDL=True
 except Exception as E:
     print(E)
     fastDL=False
 
-archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/W7X/"
-
-portcamdict={
-    'OP1.2a':{
-        'AEF10': 'IRCam_Caleo768kL_0901',
-        'AEF11': 'IRCam_Caleo768kL_0906',
-        'AEF20': 'IRCam_Caleo768kL_0702',
-        'AEF21': 'IRCam_Caleo768kL_0904',
-        'AEF30': 'IRCam_Caleo768kL_0907',
-        'AEF31': 'IRCam_Caleo768kL_0903',
-        'AEF40': 'IRCam_Caleo768kL_0701',
-        'AEF41': 'IRCam_Caleo768kL_0902',
-        'AEF50': 'Infratec_9312907',
-        'AEF51': 'IRCam_Caleo768kL_0905'}
-    	}
 
 try:
     config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\v1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
@@ -50,146 +35,6 @@ try:
 except:
     config_path=""
 
-
-
-camlimdict={
-    'IRCam_Caleo768kL_0901':
-        {
-        1:(50,5000),
-        2:(50,3000),
-        3:(50,2050),
-        4:(50,1200),
-        5:(50,1100),
-        6:(50,850),
-        7:(50,800),
-        8:(50,600),
-        9:(50,550)
-         },
-    'IRCam_Caleo768kL_0902':
-        {
-        1:(50,5000),
-        2:(50,3000),
-        3:(50,2050),
-        4:(50,1200),
-        5:(50,1100),
-        6:(50,1000),
-        7:(50,800),
-        8:(50,800),
-        9:(50,650)
-         },
-    'IRCam_Caleo768kL_0903':
-        {
-        1:(50,5000),
-        2:(50,3000),
-        3:(50,2050),
-        4:(50,1200),
-        5:(50,1100),
-        6:(50,1000),
-        7:(50,900),
-        8:(50,800),
-        9:(50,700)
-         },
-    'IRCam_Caleo768kL_0904':
-        {
-        1:(50,5000),
-        2:(50,3000),
-        3:(50,2050),
-        4:(50,1200),
-        5:(50,1100),
-        6:(50,900),
-        7:(50,800),
-        8:(50,700),
-        9:(50,600)
-         },
-    'IRCam_Caleo768kL_0905':
-        {
-        1:(50,5000),
-        2:(50,3000),
-        3:(50,2050),
-        4:(50,1200),
-        5:(50,1100),
-        6:(50,900),
-        7:(50,800),
-        8:(50,700),
-        9:(50,600)
-         },
-    'IRCam_Caleo768kL_0906':
-        {
-        1:(50,5000),
-        2:(50,3000),
-        3:(50,2050),
-        4:(50,1200),
-        5:(50,1000),
-        6:(50,800),
-        7:(50,750),
-        8:(50,600),
-        9:(50,550)
-         },
-    'IRCam_Caleo768kL_0907':
-        {
-        1:(50,5000),
-        2:(50,3000),
-        3:(50,2050),
-        4:(50,1200),
-        5:(50,1100),
-        6:(50,90),
-        7:(50,800),
-        8:(50,700),
-        9:(50,600)
-         },
-    'IRCam_Caleo768kL_0701':        
-        {
-        1:(50,5000),
-        2:(50,3000),
-        3:(50,2050),
-        4:(50,1580),
-        5:(50,1100),
-        6:(50,900),
-        7:(50,800),
-        8:(50,750),
-        9:(50,700)
-         },
-    'IRCam_Caleo768kL_0702':
-        {
-        1:(50,5000),
-        2:(50,3000),
-        3:(50,2050),
-        4:(50,1580),
-        5:(50,1150),
-        6:(50,1000),
-        7:(50,800),
-        8:(50,750),
-        9:(50,700)
-         },
-     'Infratec_9312907':
-         {
-             0:{
-             25:(50,400),
-             50:(50,300),
-            100:(50,250),
-            200:(50,200),
-            400:(50,150),
-            800:(50,100)
-             },
-             1:{
-             25:(300,1200),
-             50:(200,800),
-            100:(150,650),
-            200:(100,500),
-            400:(50,400),
-            800:(50,300)
-             },
-            2:{
-             25:(200,1200),
-             50:(200,1200),
-            100:(200,1200),
-            200:(200,1200),
-            400:(200,1000),
-            800:(200,700)
-             },
-         }
-}
-
 def get_OP_by_program(program):
     try:    
         bla=int(program.split(".")[0])
@@ -218,6 +63,28 @@ def get_OP_by_time(time_ns):
         else:
             return None
 
+def get_latest_version(stream):
+     """Find out latest version of given stream
+     :param stream   url stream of interest
+     :return int of version number of None if non-versioned or non-existing stream
+     author: G. Schlisio, edit by holn
+     """
+     request =urllib.request.Request(archivepath + stream + "/_versions.json", headers={"Accept": "application/json"})
+     try:
+         response = urllib.request.urlopen(request)
+         d = json.loads(response.read().decode('utf-8'))
+     except urllib.error.HTTPError as ex:
+         msg = ex.read()
+         raise RuntimeError(msg)
+     # detect unversioned or non-existing stream
+     if d["versionInfo"] == []:
+         return None
+     versions = []
+     for i in d['versionInfo']:
+         versions.append(i['number'])
+
+     return max(versions)
+
 def TimeToNs(date,time):
     """
     TimeToNs(date,time)
@@ -259,7 +126,7 @@ def read_restdb_old(request_url):
         valid: access ok
         t: numpy-array of time
         signal: numpy-array of requested signals
-    by Thomsen
+    by H. Thomsen
     """
     try:
         res = urllib.request.urlopen(request_url)
@@ -275,27 +142,37 @@ def read_restdb_old(request_url):
         t=np.array(signal_list['dimensions'])
         return True, t, signal0
 
-def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
+def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0):
     """
     download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
         time in ns
+    Have to swap 11, 21 until correction in the database
     """    
+    if port == 21:
+        port = 11
+    elif port == 11:
+        port = 21
     if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
-        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
+        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
+        larchivepath=archivepath+stream#"QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
         query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
     elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
-        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_LUT_"
+        stream="QRT_IRCAM/"+"AEF"+str(port)+"_LUT_"
+        larchivepath=archivepath+stream
         query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
     else:
         print("camera unknown, stopping here")
         raise Exception
+    if version==0:
+        version=get_latest_version(stream+"DATASTREAM")
+        print("LUT V"+str(version)+" is used")
     #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
     LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
     if LUTpar[0]:
         LUTid=LUTpar[2][0]['structure'][query]
-        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
+        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
         if LUTs[0]:
-            LUTs=LUTs[2][0]
+            LUTs=LUTs[2].swapaxes(1,2)[0]
             LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
             del LUTpar, LUTs
             return True,LUT
@@ -308,19 +185,25 @@ def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
         print("unable to find LUTs, check your request")
         return False,0
 
-def download_NUC_by_program(port,program,exposure,version=1):
+def download_NUC_by_program(port,program,exposure,version=0):
     prog=AKF_1.get_program_from_PID(program)
+#    try:
+#        t_program = AKF_2.get_program_from_to(program)
+#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
     if prog[0]:
         starttime=prog[1]['trigger']['0'][0]
         stoptime=prog[1]['trigger']['1'][0]
         return download_NUC_by_times(port,starttime,stoptime,exposure,version)
     else:
+#    except:
         print("cannot find the program")
         return False,0,0
 
-def download_NUC_by_times(port,starttime,stoptime,exposure,version=1):
+def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
     larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_NUC_"
 #    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
+    if version==0:
+        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_NUC_DATASTREAM")
     try:
         res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
         signal_list = json.loads(res.read().decode('utf-8'))
@@ -352,24 +235,34 @@ def download_NUC_by_times(port,starttime,stoptime,exposure,version=1):
     else:
         return False,0,0
     
-def download_background_by_program(port,program,exposure,version=1):
-    prog=AKF_1.get_program_from_PID(program)
-    if prog[0]:
-        starttime=prog[1]['trigger']['0'][0]
-        stoptime=prog[1]['trigger']['1'][0]
+def download_background_by_program(port,program,exposure,version=0):
+    '''
+    returned exist,time,frame
+    '''
+#    prog=AKF_1.get_program_from_PID(program)
+    try:
+        t_program = AKF_2.get_program_from_to(program)
+        prog =AKF_2.get_program_list(t_program[0], t_program[1])
+#    if prog[0]:
+        starttime=prog[0]['trigger']['0'][0]
+        stoptime=prog[0]['trigger']['1'][0]
         return download_background_by_times(port,starttime,stoptime,exposure,version)
-    else:
+    except:
         print("cannot find the program")
         return False,0,0
     
-def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=1):
+def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
     if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_background_" 
         larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_background_"      
     elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+        stream="QRT_IRCAM/"+"AEF"+str(port)+"_background_"   
         larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_background_"   
     else:
         print("camera unknown, stopping here")
         raise Exception
+    if version==0:
+        version=get_latest_version(stream+"DATASTREAM")
     try:
         res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
         signal_list = json.loads(res.read().decode('utf-8'))
@@ -402,11 +295,14 @@ def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=
     else:
         return False,0,0
 
-def download_raw_images_by_program(port,program,time_s=0,version=1):
-    prog=AKF_1.get_program_from_PID(program)
-    if prog[0]:
-        starttime=prog[1]['trigger']['1'][0]
-        stoptime=prog[1]['trigger']['6'][0]
+def download_raw_images_by_program(port,program,time_s=0,version=0):
+#    prog=AKF_1.get_program_from_PID(program)
+    try:
+        t_program = AKF_2.get_program_from_to(program)
+        prog =AKF_2.get_program_list(t_program[0], t_program[1])
+#    if prog[0]:
+        starttime=prog[0]['trigger']['1'][0]
+        stoptime=prog[0]['trigger']['6'][0]
         if type(time_s)==list:
             tstart=time_s[0]
             tstop=time_s[1]
@@ -418,14 +314,19 @@ def download_raw_images_by_program(port,program,time_s=0,version=1):
                 return download_raw_images_by_times(port,starttime,stoptime,version)
             else:
                 return download_raw_images_by_times(port,starttime,int(starttime+time_s*1e9),version)
-    else:
+    except:
         print("cannot find the program")
         return False,0,0
 
-def download_raw_images_by_times(port,starttime,stoptime,version=1,intervalSize=1E9):
+def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9):
+    
     if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+        if version==0:
+            version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
         larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
     elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+        if version==0:
+            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
         larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
     else:
         print("camera unknown, stopping here")
@@ -472,17 +373,24 @@ def download_raw_images_by_times(port,starttime,stoptime,version=1,intervalSize=
             return False, 0,-1
 
 if fastDL:
-    def download_raw_images_by_program_via_png(port,program,time_s=0,version=1,threads=1):
-        prog=AKF_1.get_program_from_PID(program)
-        if prog[0]:
-            starttime=prog[1]['trigger']['1'][0]
-            stoptime=prog[1]['trigger']['6'][0]
+    def download_raw_images_by_program_via_png(port,program,time_s=0,version=0,threads=1):
+#        prog=AKF_1.get_program_from_PID(program)
+        try:
+            t_program = AKF_2.get_program_from_to(program)
+            prog =AKF_2.get_program_list(t_program[0], t_program[1])
+#        if prog[0]:
+            starttime=prog[0]['trigger']['1'][0]
+            stoptime=prog[0]['trigger']['6'][0]
             success=True
             OP=get_OP_by_time(starttime)
             Cam=portcamdict[OP]['AEF'+str(port)]
             if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+                if version==0:
+                    version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
                 larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
             elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+                if version==0:
+                    version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
                 larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
             else:
                 raise Exception("Port number does not fit the known cameras")
@@ -505,7 +413,7 @@ if fastDL:
                     enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)  
                     enddate=enddate.isoformat()
                 #"2017-11-15 08:00:00"
-            times=AKF_2.get_all_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))
+            times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
             time=[]
             images=[]
             lnt=len(times)
@@ -554,29 +462,82 @@ if fastDL:
                     times=times+resultdict[order.index(i)][2]
                 del resultdict
                 return success,np.array(times),images
+        except Exception as E:
+            raise Exception(E)
             
     
-    def download_raw_images_png_by_times_thread(port,times,out_q,threadnumber,version=1):
+    def download_raw_images_png_by_times_thread(port,times,out_q,threadnumber,version=0):
         images=[]
         time=[]
         successes=[]
         for i in times:
-            imag=download_last_raw_image_by_time(port,i-10,i+10)
+            imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
             if imag[0]:
                 images.append(np.array(imag[1],dtype=np.uint16))
+#                print(np.min(imag[1]),np.max(imag[1]))
                 time.append(i)
                 successes.append(True)
             else:
                 successes.append(False)
         out_q.put([threadnumber,successes,time,images])        
     #    return success,np.array(time),np.array(images,dtype=np.uint16)
+        
+    def convert_raw_to_temp_thread(port,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
+        images=[]
+        time=[]
+        successes=[]
+        for i in times:
+            imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
+            if imag[0]:
+                images.append(np.array(imag[1],dtype=np.uint16))
+                time.append(i)
+                successes.append(True)
+            else:
+                successes.append(False)
+        if give_ERROR:
+            images,error_images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
+        else:
+            images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
+        
+        for i in range(len(images)):
+            images[i]=(images[i]*FOV).astype(np.float32)
+        print(datetime.datetime.now(),"correcting bad pixels")
+        images=IR_tools.correct_images(images,badpixels)
+        out_q.put([threadnumber,successes,time,images]) 
     
-def download_last_raw_image_by_time(port,starttime,stoptime,version=1):
+    def NUC_raw_thread(port,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
+        images=[]
+        time=[]
+        successes=[]
+        for i in times:
+            imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
+            if imag[0]:
+                images.append(np.array(imag[1],dtype=np.uint16))
+                time.append(i)
+                successes.append(True)
+            else:
+                successes.append(False)
+        if give_ERROR:
+            images,error_images=apply_NUC(images,gain,offset,gain_error,offset_error,give_ERROR)
+        else:
+            images=apply_NUC(images,gain,offset,gain_error,offset_error,give_ERROR)
+        
+        for i in range(len(images)):
+            images[i]=(images[i]*FOV).astype(np.float32)
+        print(datetime.datetime.now(),"correcting bad pixels")
+        images=IR_tools.correct_images(images,badpixels)
+        out_q.put([threadnumber,successes,time,images]) 
+        
+def download_last_raw_image_by_time(port,starttime,stoptime,version=0):
     OP=get_OP_by_time(starttime)
     Cam=portcamdict[OP]['AEF'+str(port)]
     if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+        if version==0:
+            version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
         larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
     elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+        if version==0:
+            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
         larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
     else:
         raise Exception("camera unknown, stopping here")
@@ -591,20 +552,27 @@ def download_last_raw_image_by_time(port,starttime,stoptime,version=1):
         print(e)
         return False, -1
     
-def download_raw_parlog_by_program(port,program,version=1):
-    prog=AKF_1.get_program_from_PID(program)
-    if prog[0]:
-        starttime=prog[1]['trigger']['1'][0]
-        stoptime=prog[1]['trigger']['6'][0]
+def download_raw_parlog_by_program(port,program,version=0):
+#    prog=AKF_1.get_program_from_PID(program)
+    try:
+        t_program = AKF_2.get_program_from_to(program)
+        prog =AKF_2.get_program_list(t_program[0], t_program[1])
+#    if prog[0]:
+        starttime=prog[0]['trigger']['1'][0]
+        stoptime=prog[0]['trigger']['6'][0]
         return download_raw_parlog_by_times(port,starttime,stoptime,version)    
-    else:
+    except:
         print("cannot find the program")
         return False,0,0
     
-def download_raw_parlog_by_times(port,starttime,stoptime,version=1):
+def download_raw_parlog_by_times(port,starttime,stoptime,version=0):
     if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+        if version==0:
+            version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG")
         larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)
     elif port in [10,11,20,21,30,31,40,41,51]:#elif camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+        if version==0:
+            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG")
         larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)   
     else:
         print("camera unknown, stopping here")
@@ -621,17 +589,22 @@ def download_raw_parlog_by_times(port,starttime,stoptime,version=1):
         return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']
     
 
-def get_INFRATEC_filter_by_program(program,version=1):
-    prog=AKF_1.get_program_from_PID(program)
-    if prog[0]:
-        starttime=prog[1]['trigger']['1'][0]
-        stoptime=prog[1]['trigger']['6'][0]
+def get_INFRATEC_filter_by_program(program,version=0):
+#    prog=AKF_1.get_program_from_PID(program)
+    try:
+        t_program = AKF_2.get_program_from_to(program)
+        prog =AKF_2.get_program_list(t_program[0], t_program[1])
+#    if prog[0]:
+        starttime=prog[0]['trigger']['1'][0]
+        stoptime=prog[0]['trigger']['6'][0]
         return  get_INFRATEC_filter_by_times(starttime,stoptime,50,version)
-    else:
+    except:
         print("cannot find the program")
         return False,0,0
 
-def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=1):
+def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0):
+    if version==0:
+        version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG")
     larchivepath=archivepath+"QRT_INFRATEC/AEF"+str(port)+"_raw_PARLOG/V"+str(version)
     try:
         res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
@@ -646,79 +619,108 @@ def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=1):
     else:
         return False, 0,-1
     
-def get_exposure_by_program(port,program,version=1):
+def get_exposure_by_program(port,program,version=0):
     prog=AKF_1.get_program_from_PID(program)
+#    try:
+#        t_program = AKF_2.get_program_from_to(program)
+#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
     if prog[0]:
         starttime=prog[1]['trigger']['1'][0]
         stoptime=prog[1]['trigger']['6'][0]
         return get_exposure_by_times(port,starttime,stoptime,version)
     else:
-        print("cannot find the program")
+#    except Exception as E:
+        print("cannot find the program, no exposure time available;")#, E)
         return False,0,0
         
-def get_exposure_by_times(port,starttime,stoptime,version=1):
+def get_exposure_by_times(port,starttime,stoptime,version=0):
     if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
-        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
+        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"   
+        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_" 
     elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
-        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"   
+        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_" 
+        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
+    if version==0:
+        version=get_latest_version(stream+"DATASTREAM")
     return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/3/exposuretime"+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
       
-def get_camera_temp_by_program(port,program,version=1):
-    prog=AKF_1.get_program_from_PID(program)
-    if prog[0]:
-        starttime=prog[1]['trigger']['1'][0]
-        stoptime=prog[1]['trigger']['6'][0]
+def get_camera_temp_by_program(port,program,version=0):
+#    prog=AKF_1.get_program_from_PID(program)
+    try:
+        t_program = AKF_2.get_program_from_to(program)
+        prog =AKF_2.get_program_list(t_program[0], t_program[1])
+#    if prog[0]:
+        starttime=prog[0]['trigger']['1'][0]
+        stoptime=prog[0]['trigger']['6'][0]
         return get_camera_temp_by_times(port,starttime,stoptime,version)
-    else:
+    except:
         print("cannot find the program")
         return False,0,0
         
-def get_camera_temp_by_times(port,starttime,stoptime,version=1):
+def get_camera_temp_by_times(port,starttime,stoptime,version=0):
     if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"  
         larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
     elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
         larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
     else:
         print("Port unknown")
         raise Exception
+    if version==0:
+        version=get_latest_version(stream+"DATASTREAM")
     dummy=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
     if dummy[0]:
         return dummy
     else:
         return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratempetarure/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
     
-def get_sensor_temp_by_program(port,program,version=1):
-    prog=AKF_1.get_program_from_PID(program)
-    if prog[0]:
-        starttime=prog[1]['trigger']['1'][0]
-        stoptime=prog[1]['trigger']['6'][0]
+def get_sensor_temp_by_program(port,program,version=0):
+#    prog=AKF_1.get_program_from_PID(program)
+    try:
+        t_program = AKF_2.get_program_from_to(program)
+        prog =AKF_2.get_program_list(t_program[0], t_program[1])
+#    if prog[0]:
+        starttime=prog[0]['trigger']['1'][0]
+        stoptime=prog[0]['trigger']['6'][0]
         return get_sensor_temp_by_times(port,starttime,stoptime,version)
-    else:
+    except:
         print("cannot find the program")
         return False,0,0
         
-def get_sensor_temp_by_times(port,starttime,stoptime,version=1):
+def get_sensor_temp_by_times(port,starttime,stoptime,version=0):
     if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
         larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
     elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
         larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
+    if version==0:
+        version=get_latest_version(stream+"DATASTREAM")
     return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
 
-def get_camera_framerate_by_program(port,program,version=1):
-    prog=AKF_1.get_program_from_PID(program)
-    if prog[0]:
-        starttime=prog[1]['trigger']['1'][0]
-        stoptime=prog[1]['trigger']['6'][0]
+def get_camera_framerate_by_program(port,program,version=0):
+#    prog=AKF_1.get_program_from_PID(program)
+    try:
+        t_program = AKF_2.get_program_from_to(program)
+        prog =AKF_2.get_program_list(t_program[0], t_program[1])
+#    if prog[0]:
+        starttime=prog[0]['trigger']['1'][0]
+        stoptime=prog[0]['trigger']['6'][0]
         return get_camera_framerate_by_times(port,starttime,stoptime,version)
-    else:
+    except:
         print("cannot find the program")
         return False,0,0
         
-def get_camera_framerate_by_times(port,starttime,stoptime,version=1):
+def get_camera_framerate_by_times(port,starttime,stoptime,version=0):
     if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
         larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
     elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
         larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
+    if version==0:
+        version=get_latest_version(stream+"DATASTREAM")    
     return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
     
 
@@ -795,42 +797,40 @@ def get_average_divertor_TC_for_camtemp(port,starttime,endtime,part="all",TC=0):
                 success=False
         try:
             return success,time,signal/n
-        except:
+        except Exception as E:
+            print(E)
             return False,0,-1
     else:
+        print("same numbers!")
         return False,0,-1
 
+def get_divertor_TC_coordinate(cameraport,TC=0):
+    from IR_config_constants import TC_database
+    if cameraport %10==1:##uneven cameraport, even TC port
+        portnr=cameraport-1
+    elif cameraport%10==0:##even cameraport, uneven TC port
+        portnr=cameraport+1
+    else:        
+        raise Exception("unknown port!")    
+    return TC_database[portnr][TC]
+
 def get_TC_data_for_port_by_program(port,program,part=0,version=0):
-    prog=AKF_1.get_program_from_PID(program)
-    if prog[0]:
-        starttime=prog[1]['trigger']['0'][0]
-        stoptime=prog[1]['trigger']['1'][0]
+#    prog=AKF_1.get_program_from_PID(program)
+    try:
+        t_program = AKF_2.get_program_from_to(program)
+        prog =AKF_2.get_program_list(t_program[0], t_program[1])
+#    if prog[0]:
+        starttime=prog[0]['trigger']['1'][0]
+        stoptime=prog[0]['trigger']['6'][0]
         return get_TC_data_for_port(port,starttime,stoptime,part,version)
-    else:
+    except:
         print("cannot find the program")
         return False,0,0
         
 def get_TC_data_for_port(port,starttime,endtime,part=0,version=0):
     urlstart="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.115/DataModuleDesc.19471_DATASTREAM/"
     #66,67(10),72,73(20),78,79(30),84,85(40/50_2),90,91(50_1)
-    TC_port={
-    10:[10],
-    11:[10],
-    20:[20],
-    21:[20],
-    30:[30],
-    31:[30],
-    40:[10,20,30],
-    41:[10,20,30],
-    50:[50],
-    51:[10,20,30]
-    }
-    TC_channel={
-    10:[[66,67],["ActV1QSR10CT001","ActV1QSR10CT002"]], 
-    20:[[72,73],["ActV1QSR20CT001","ActV1QSR20CT002"]],
-    30:[[78,79],["ActV1QSR30CT001","ActV1QSR30CT002"]],
-    50:[[90,91,84,85],["ActV1QRT50CT001","ActV1QRT50CT002","ActV1QRT50CT003","ActV1QRT50CT004"]]
-    }
+    
     n=0
     success=True
     signal=0
@@ -894,7 +894,10 @@ def terror_from_LUT(LUT,images):
 def check_temperature_range(time,maxvalue_C,minvalue_C,port,exposure,cfilter=0):
     try:
         cam=portcamdict[get_OP_by_time(time)]['AEF'+str(port)]
-        ranges=camlimdict[cam][exposure]
+        if port==50:
+            ranges=camlimdict[cam][cfilter][exposure]
+        else:
+            ranges=camlimdict[cam][exposure]
 #        if minvalue_C>=ranges[0] and maxvalue_C<=ranges[1]:
 #            return True
         if minvalue_C<ranges[0] and maxvalue_C<=ranges[1]:
@@ -910,13 +913,16 @@ def check_temperature_range(time,maxvalue_C,minvalue_C,port,exposure,cfilter=0):
         raise Exception(E)
         
 
-def get_temp_from_raw_by_program_V1(portnr,program,time_s=0,emi=0.8,divertorpart="all",version=1,intervalSize=1E9,give_ERROR=False):
+def get_temp_from_raw_by_program_V1(portnr,program,time_s=0,emi=0.8,divertorpart="all",version=0,intervalSize=1E9,give_ERROR=False):
     background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,1,version)    
-    prog=AKF_1.get_program_from_PID(program)
-    if prog[0]:
+#    prog=AKF_1.get_program_from_PID(program)
+    try:
+        t_program = AKF_2.get_program_from_to(program)
+        prog =AKF_2.get_program_list(t_program[0], t_program[1])
+#    if prog[0]:
 #        t0=prog[1]['trigger']['0'][0]
-        t1=prog[1]['trigger']['1'][0]
-        t6=prog[1]['trigger']['6'][0]
+        t1=prog[0]['trigger']['1'][0]
+        t6=prog[0]['trigger']['6'][0]
         starttime=t1-10
         if type(time_s)==list:
             tstart=time_s[0]
@@ -982,15 +988,16 @@ def get_temp_from_raw_by_program_V1(portnr,program,time_s=0,emi=0.8,divertorpart
             return success,times,temperatureimages,valid,error_images
         else:
             return success,times,temperatureimages,valid          
-    else:
+    except:
         print("cannot find program")
         return False,0,-1,False
 
-def get_temp_from_raw_by_program_V2(portnr,program,time_s=0,emi=0.8,version=1,threads=1,give_ERROR=False):
-    return get_temp_from_raw_by_program(portnr,program,time_s=time_s,emi=emi,T_version=2,version=version,threads=threads,give_ERROR=give_ERROR)
+def get_temp_from_raw_by_program_V2(portnr,program,time_s=0,emi=0.8,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False):
+    return get_temp_from_raw_by_program(portnr,program,time_s=time_s,emi=emi,T_version=2,version=version,threads=threads,give_ERROR=give_ERROR,use_firstframe_as_background=use_firstframe_as_background)
   
-def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,version=1,threads=1,give_ERROR=False):
-    background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version)
+def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,back_emissivity=0.8):
+    background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version,back_emissivity)
+    print(datetime.datetime.now(),"raw download start")
     if fastDL: 
         exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_s,version,threads)
     else:
@@ -1001,6 +1008,9 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
             raise Exception("no data found")
     print(datetime.datetime.now(),"raw download finished")
     FOV=get_FOV_mask(portnr)
+    if use_firstframe_as_background:
+        bim=apply_NUC([frames[0]],gain,offset)
+        background=get_average_background_recangle(portnr,bim[0])
     if give_ERROR:
         frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
     else:
@@ -1009,35 +1019,254 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
     for i in range(len(frames)):
         frames[i]=(frames[i]*FOV).astype(np.float32)
     print(datetime.datetime.now(),"correcting bad pixels")
-    frames=correct_images(frames,badpixels)
+    frames=IR_tools.correct_images(frames,badpixels)
     print(datetime.datetime.now(),"checking temperaturerange")
     valid=True
-    for i in range(len(frames)):
-        try:
-            check_temperature_range(time[0],np.max(frames[i])-273.15,np.min(frames[i][np.nonzero(frames[i])])-273.15,portnr,t_exp,cfilter)
-        except Exception as E:
-            print(E)
-            valid=False
-            i=len(frames)
+#    for i in range(len(frames)):
+    try:
+        check_temperature_range(time[0],np.max(frames)-273.15,np.min(frames[:][np.nonzero(frames[i])])-273.15,portnr,t_exp,cfilter)
+    except Exception as E:
+        print(E)
+        valid=False
+#        i=len(frames)
     if give_ERROR:
         return exist,time,frames,valid,error_images
     else:
         return exist,time,frames,valid
 
+def get_temp_from_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,check_range=True):
+    if threads==1 or not fastDL:
+        return get_temp_from_raw_by_program(portnr,program,time_s,emi,T_version,version,threads,give_ERROR)
+    else:
+        background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version)
+        FOV=get_FOV_mask(portnr)        
+        prog=AKF_1.get_program_from_PID(program)
+        if prog[0]:
+            starttime=prog[1]['trigger']['1'][0]
+            stoptime=prog[1]['trigger']['6'][0]
+            success=True
+            OP=get_OP_by_time(starttime)
+            Cam=portcamdict[OP]['AEF'+str(portnr)]
+            if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+                if version==0:
+                    version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(portnr)+"_raw_DATASTREAM")
+                larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+            elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+                if version==0:
+                    version=get_latest_version("QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM")
+                larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+            else:
+                raise Exception("Port number does not fit the known cameras")
+            if type(time_s)==list:
+                tstart=time_s[0]
+                tstop=time_s[1]
+                if tstop<tstart:
+                    raise Exception("endtime before starttime")
+                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
+                stdate=stdate.isoformat()
+                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
+                enddate=enddate.isoformat()
+            else:
+                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
+                stdate=stdate.isoformat()
+                if time_s==0:
+                    enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
+                    enddate=enddate.isoformat()
+                else:
+                    enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)  
+                    enddate=enddate.isoformat()        
+        
+            times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
+            lnt=len(times)
+            tim=[]
+            for i in range(lnt):
+                tim.append(times[lnt-1-i][0])
+            intervalls=[]
+            intervalSize=int(lnt/threads)
+            for i in range(threads):
+                intervalls.append(int(i*intervalSize))
+            intervalls.append(lnt)
+            jobs = []
+            out_q=multiprocessing.Queue()
+            for i in range(threads):
+                print("Start Thread ",i+1)                
+                p = multiprocessing.Process(target=convert_raw_to_temp_thread, args=(portnr,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,))
+                jobs.append(p)
+                p.start()
+            resultdict = []
+            for i in range(threads):
+                resultdict.append(out_q.get())
+            for p in jobs:
+                p.join()
+            print("all threads are done")
+            order=[]
+            for ele in resultdict:
+                order.append(ele[0])
+                if len(np.where(np.asarray(ele[1])==False)[0])>0:
+                    success=False            
+            times=[]#np.array(resultdict[order.index(0)][2])
+            images=[]#np.array(resultdict[order.index(0)][3])
+            for i in range(threads):
+                images=images+resultdict[order.index(i)][3]
+                times=times+resultdict[order.index(i)][2]
+            del resultdict
+            if check_range:
+                print(datetime.datetime.now(),"checking temperaturerange")
+                valid=True
+                for i in range(len(images)):
+                    try:
+                        check_temperature_range(times[0],np.max(images[i])-273.15,np.min(images[i][np.nonzero(images[i])])-273.15,portnr,t_exp,cfilter)
+                    except Exception as E:
+                        print(E)
+                        valid=False
+                        i=len(images)
+            else:
+                valid=True
+        else:
+            success=False
+            times=[0]
+            images=[0]
+            valid=False
+            error_images=[0]
+        
+        if give_ERROR:
+            return success,np.array(times),images,valid,error_images
+        else:
+            return success,np.array(times),images,valid
+
+def get_nuced_raw_by_program(portnr,program,time_s=0,version=0,threads=1,give_ERROR=False):
+    background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi=1,T_version=2,version=version)
+    print(datetime.datetime.now(),"raw download start")
+    if fastDL: 
+        exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_s,version,threads)
+    else:
+        exist,time,frames=download_raw_images_by_program(portnr,program,time_s,version)
+        if exist:
+            frames=[im.swapaxes(0,1) for im in frames]
+        else:
+            raise Exception("no data found")
+    print(datetime.datetime.now(),"raw download finished")
+    FOV=get_FOV_mask(portnr)
+    if give_ERROR:
+        frames,frames_error=apply_NUC(frames,gain,offset,gain_error,offset_error,give_ERROR)
+    else:
+        frames=apply_NUC(frames,gain,offset,gain_error,offset_error,give_ERROR)
+    
+    for i in range(len(frames)):
+        frames[i]=(frames[i]*FOV).astype(np.float32)
+    print(datetime.datetime.now(),"correcting bad pixels")
+    frames=IR_tools.correct_images(frames,badpixels)
+    print(datetime.datetime.now(),"checking temperaturerange")
+    valid=True
+
+    if give_ERROR:
+        return exist,time,frames,valid,frames_error
+    else:
+        return exist,time,frames,valid
 
-def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,fullbackground=False,give_error=False):
+def get_nuced_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,check_range=True):
+    if threads==1 or not fastDL:
+        return get_nuced_raw_by_program(portnr,program,time_s,emi,T_version,version,threads,give_ERROR)
+    else:
+        background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version)
+        FOV=get_FOV_mask(portnr)        
+        prog=AKF_1.get_program_from_PID(program)
+        if prog[0]:
+            starttime=prog[1]['trigger']['1'][0]
+            stoptime=prog[1]['trigger']['6'][0]
+            success=True
+            OP=get_OP_by_time(starttime)
+            Cam=portcamdict[OP]['AEF'+str(portnr)]
+            if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+                if version==0:
+                    version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(portnr)+"_raw_DATASTREAM")
+                larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+            elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+                if version==0:
+                    version=get_latest_version("QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM")
+                larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+            else:
+                raise Exception("Port number does not fit the known cameras")
+            if type(time_s)==list:
+                tstart=time_s[0]
+                tstop=time_s[1]
+                if tstop<tstart:
+                    raise Exception("endtime before starttime")
+                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
+                stdate=stdate.isoformat()
+                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
+                enddate=enddate.isoformat()
+            else:
+                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
+                stdate=stdate.isoformat()
+                if time_s==0:
+                    enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
+                    enddate=enddate.isoformat()
+                else:
+                    enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)  
+                    enddate=enddate.isoformat()        
+        
+            times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
+            lnt=len(times)
+            tim=[]
+            for i in range(lnt):
+                tim.append(times[lnt-1-i][0])
+            intervalls=[]
+            intervalSize=int(lnt/threads)
+            for i in range(threads):
+                intervalls.append(int(i*intervalSize))
+            intervalls.append(lnt)
+            jobs = []
+            out_q=multiprocessing.Queue()
+            for i in range(threads):
+                print("Start Thread ",i+1)                
+                p = multiprocessing.Process(target=NUC_raw_thread, args=(portnr,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,))
+                jobs.append(p)
+                p.start()
+            resultdict = []
+            for i in range(threads):
+                resultdict.append(out_q.get())
+            for p in jobs:
+                p.join()
+            print("all threads are done")
+            order=[]
+            for ele in resultdict:
+                order.append(ele[0])
+                if len(np.where(np.asarray(ele[1])==False)[0])>0:
+                    success=False            
+            times=[]#np.array(resultdict[order.index(0)][2])
+            images=[]#np.array(resultdict[order.index(0)][3])
+            for i in range(threads):
+                images=images+resultdict[order.index(i)][3]
+                times=times+resultdict[order.index(i)][2]
+            del resultdict
+            valid=True
+        else:
+            success=False
+            times=[0]
+            images=[0]
+            valid=False
+            error_images=[0]
+        
+        if give_ERROR:
+            return success,np.array(times),images,valid,error_images
+        else:
+            return success,np.array(times),images,valid        
+
+def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,fullbackground=False,give_ERROR=False):
     try:
         #images=np.array(raw,dtype=np.uint16)
 #        del raw
 #        images=images.swapaxes(1,2)
         if type(gain)!=int and type(offset)!=int:
             print(datetime.datetime.now(),"NUCing")
-            if give_error:
-                images,error_images=apply_NUC(images,gain,offset,gain_error,offset_error,give_error)
+            if give_ERROR:
+                images,error_images=apply_NUC(images,gain,offset,gain_error,offset_error,give_ERROR)
             else:
                 images=apply_NUC(images,gain,offset)
         print(datetime.datetime.now(),"background treatment")
 #        if fullbackground:
+        
         for i in range(len(images)):
             images[i]=(images[i]-background).astype(np.float32)
 #        else:
@@ -1047,7 +1276,7 @@ def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_erro
         images=[np.round(im.clip(min=0)).astype(np.uint16) for im in images]
         print(datetime.datetime.now(),"applying LUT")
         LUT=np.array([LUT[1],LUT[2]])
-        if give_error:
+        if give_ERROR:
             terror=[]#np.zeros(np.shape(images))
             for i in range(len(images)):                 
                 # uncertainty of temperature calibration
@@ -1086,15 +1315,19 @@ def apply_NUC(images,gain,offset,gain_error=0,offset_error=0,give_error=False):
                 images[i]=(images[i]+offset).astype(np.float32)        
             return images
     except Exception as E:
+        print("error")
         print(E)
         return 0
         
-def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=1):
-    prog=AKF_1.get_program_from_PID(program)
-    if prog[0]:
-        t0=prog[1]['trigger']['0'][0]
-        t1=prog[1]['trigger']['1'][0]
-        t6=prog[1]['trigger']['6'][0]
+def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivity=0.8):
+#    prog=AKF_1.get_program_from_PID(program)
+    try:
+        t_program = AKF_2.get_program_from_to(program)
+        prog =AKF_2.get_program_list(t_program[0], t_program[1])
+#    if prog[0]:
+        t0=prog[0]['trigger']['0'][0]
+        t1=prog[0]['trigger']['1'][0]
+        t6=prog[0]['trigger']['6'][0]
         OP=get_OP_by_program(program)
         Camera=portcamdict[OP]['AEF'+str(port)]
         if Camera.split("_")[0]=="Infratec":
@@ -1115,7 +1348,7 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=1):
             if LUT_DL[0]:
                 LUT=LUT_DL[1]
                 del LUT_DL
-                back_DL=download_background_by_times(port,t0,t1,t_exp,version)
+                back_DL=download_background_by_times(port,t0,t1,t_exp,cfilter,version)
                 if back_DL[0]:
                     background=back_DL[2]
                     if Temp_V==1:
@@ -1131,25 +1364,79 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=1):
                                 refT=np.average(TCT_dl[2])
                             else:
                                 raise Exception("Unable to find thermocouple data")
-                    else:
+                    elif Temp_V==2:
+                        frame=background.copy()
+                        background=get_average_background_recangle(port,background)
+                        refT=28.5
+                    elif Temp_V==3:
+                        frame=background.copy()
                         background=get_average_background_recangle(port,background)
                         refT=28.5
-                    if cfilter==0:
-                        NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
-                        if NUC_DL[0]==False:
-                            raise Exception("NUC was not found")
+                        print("test of V3")
+#                        back_off=estimate_offset(port,program)                      
+#                        background=(background-back_off)/(back_emissivity)+back_off
+                    else:
+                        raise Exception("Unknown temperature calibration method")
+                    if cfilter==0 and not Camera.split("_")[0]=="Infratec":
+                        if (port == 31) or (port == 21 and float(program[4:]) > 1110):
+                            print("rebuilding coldframe")
+                            #use any way the rebuilt coldframe.
+                            sT = get_sensor_temp_by_program(port, program)[2][0]
+                            coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
+                            filestring = 'AEF' + str(port) + '_et' + str(t_exp)
+                            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
+                            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
+                            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
+                            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref)
+                            gain[gain == np.inf] = 0
+                            offset[offset == np.inf] = 0
+                            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
+                            gain_error=0
+                            offset_error=0
+                            exist,time,frames=download_raw_images_by_times(port,t1,int(t1+0.02*1e9),version)
+                            if exist:
+                                frames=[im.swapaxes(0,1) for im in frames]
+                                bim=apply_NUC([frames[0]],gain,offset)
+                                background=get_average_background_recangle(port,bim[0])
+                            else:
+                                raise Exception("cannot find the first frame of the discharge, reconstruction failed")
+                        elif (port == 11) or (port == 21):
+                            coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
+                            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
+                            if NUC_DL[0]==False:
+                                raise Exception("NUC was not found")
+                            else:
+                                ci = NUC_DL[1][2]
+                            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref)
+                            gain[gain == np.inf] = 0
+                            offset[offset == np.inf] = 0
+                            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)                                
+                            gain_error=0
+                            offset_error=0
+                            exist,time,frames=download_raw_images_by_times(port,t1,int(t1+0.02*1e9),version)
+                            if exist:
+                                frames=[im.swapaxes(0,1) for im in frames]
+                                bim=apply_NUC([frames[0]],gain,offset)
+                                background=get_average_background_recangle(port,bim[0])
+                            else:
+                                raise Exception("background wrong due to wrong NUC, firstframe not found, reconstruction failed")
                         else:
-                            gain=np.array(NUC_DL[1][0])
-                            offset=np.array(NUC_DL[1][1])
-                            gain_error=np.array(NUC_DL[1][4])
-                            offset_error=np.array(NUC_DL[1][5])
-                            badpixels=np.array(NUC_DL[1][3],dtype=np.ubyte)
-                            if np.max(badpixels)==0:
-                                badpixels=find_badpixels(port,gain,offset)
+                            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
+                            if NUC_DL[0]==False:
+                                raise Exception("NUC was not found")
+                            else:
+                                gain=np.array(NUC_DL[1][0])
+                                offset=np.array(NUC_DL[1][1])
+                                offset[offset<-1e300]=0
+                                gain_error=np.array(NUC_DL[1][4])
+                                offset_error=np.array(NUC_DL[1][5])
+                                badpixels=np.array(NUC_DL[1][3],dtype=np.ubyte)
+                                if np.max(badpixels)==0:
+                                    badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
                     else:
                         gain=0
                         offset=0
-                        badpixels=0#find_badpixels(port,gain,offset)
+                        badpixels=find_badpixels(port,frame-background,offset,niterations=10,tolerance=10,plot_it=False)#find_badpixels(port,gain,offset)
                         gain_error=0
                         offset_error=0
                 else:
@@ -1159,130 +1446,39 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=1):
         else:
             raise Exception("no exposure time found")
         return background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
-    else:
+    except:
         print("Program was not found")
         return 0,0,0,0,0
 
-def find_badpixels(port,gain,offset):
+def find_badpixels(port,gain,offset,niterations=3,tolerance=10,plot_it=False):
     badpixels=np.zeros(np.shape(gain))
 #    certain_bads=np.zeros(np.shape(gain))
-    gainmax=12#100
-    badpixels+=(gain>gainmax)
-    badpixels+=(gain<0.1)
+#    gainmax=12#100
+#    badpixels+=(gain>gainmax)
+#    badpixels+=(gain<0.1)
+    FOV=get_FOV_mask(port)
+    last_number=0
+    finished=False
+    n=0
+    while (n<=niterations and not finished):#len(badlist)>=last_number):      
+        badlist=IR_tools.find_outlier_pixels(gain*FOV,plot_it=plot_it,tolerance=tolerance)
+        gain=IR_tools.restore_pixels(gain*FOV,badlist)
+        print("number of found bad pixels: ",len(badlist))
+        n+=1
+        if len(badlist)>=last_number and n>2:
+            finished=True
+        last_number=len(badlist)
+        for pix in badlist:
+            badpixels[pix]=1        
     # gain*(1-certain_bads) these pixels are not certainly bad
 #    possi_goodgain=gain*(1-certain_bads)
 #    badpixels+=(gain>(np.mean(possi_goodgain)+np.std(possi_goodgain)))
 #    badpixels+=(gain<(np.mean(possi_goodgain)-np.std(possi_goodgain)))    
-    FOV=get_FOV_mask(port)
-    badpixels+=badpixels*(1-FOV)
-    return np.array(badpixels,dtype=np.ubyte)
     
-def correct_images(images,badpixels):
-    if type(badpixels)!=int:
-        for i in range(len(images)):
-            images[i]=(restore_pixels(images[i],np.invert(badpixels==1))).astype(np.float32)
-        print("done")
-    return images
-
-def restore_pixels(frame, bad_pixel):
-    # make sure bad pixel are provided as mask and list  
-    if type(bad_pixel) is list:
-        blist = bad_pixel
-        bmask = np.ones(frame.shape,dtype=bool)
-        for pix in bad_pixel:
-            bmask[pix] = False
-        bmask = np.invert(bmask)
-    else:
-        bmask = np.invert(bad_pixel)
-        x,y = np.where(bmask)
-        blist = list(zip(x,y))
-#    print('restore_pixels() working on: ',blist)
-    # prepare internal result frames
-    resframe = frame.astype(float)
-    # -----------------------------------
-    # restore by list
-    #
-    height = np.shape(frame)[0]
-    width  = np.shape(frame)[1]
-    for pos in blist:
-        # assume four neighbouring, non-bad pixels exist
-        n_neighbours = 4
-        # -- top neighbour --
-        if pos[0] == 0:
-            # out of bounds
-            top = 0
-            n_neighbours -= 1
-        else:
-            i_t = pos[0] - 1
-            while (bmask[i_t,pos[1]] & (i_t >= 0)):
-                i_t -= 1
-            if i_t >= 0:
-                top = frame[i_t,pos[1]]
-            else:
-                # out of bounds
-                top = 0
-                n_neighbours -= 1
-            #print('top: ',i_t, top)    
-        # -- bottom neighbour --
-        if pos[0] == height-1:
-            # out of bounds
-            bottom = 0
-            n_neighbours -= 1
-        else:
-            i_b = pos[0] + 1
-            while (bmask[i_b,pos[1]] & (i_b <= height-2)):
-                i_b += 1
-            if i_b >= 0:
-                bottom = frame[i_b,pos[1]]
-            else:
-                # out of bounds
-                bottom = 0
-                n_neighbours -= 1
-            #print('bottom: ',i_b, bottom)
-        # -- left neighbour --
-        if pos[1] == 0:
-            # out of bounds
-            left = 0
-            n_neighbours -= 1
-        else:
-            i_l = pos[1] - 1
-            while (bmask[pos[0],i_l] & (i_l >= 0)):
-                i_l -= 1
-            if i_l >= 0:
-                left = frame[pos[0],i_l]
-            else:
-                # out of bounds
-                left = 0
-                n_neighbours -= 1
-            #print('left: ',i_l, left)    
-        # -- right neighbour --
-        if pos[1] == width-1:
-            # out of bounds
-            right = 0
-            n_neighbours -= 1
-        else:
-            i_r = pos[1] + 1
-            while (bmask[pos[0],i_r] & (i_r <= width-2)):
-                i_r += 1
-            if i_r >= 0:
-                right = frame[pos[0],i_r]
-            else:
-                # out of bounds
-                right = 0
-                n_neighbours -= 1
-            #print('right: ',i_r, right)                
-        # averaging
-        if n_neighbours > 0:
-            #print('original value: ',frame[pos[0],pos[1]])
-            resframe[pos[0],pos[1]] = (top + bottom + left + right)/n_neighbours
-            #print('average of ',n_neighbours,' neighbours: ',frame1[pos[0],pos[1]])
-        else:
-            print('ERROR: no adjacent pixel found!')
-    return resframe
-
+    badpixels+=badpixels*((1-FOV)*2)
+    return np.array(badpixels,dtype=np.ubyte)
 
-def make_FOV_mask(port):
-    
+def make_FOV_mask(port):    
     points=valid_FOV_circle[port]
     """
     Sekante erreichnen für Kreis, Gerade ax+by=c, sonderfall: y=c, a=0,b=1
@@ -1330,7 +1526,7 @@ def get_FOV_mask(port):
     FOV=(FOV>0.1)*1
     return FOV
 
-def get_average_background_recangle(port,image):
+def get_background(port,image):
     try:
 #        points=valid_background_rectangle[port]
         FOV=get_FOV_mask(port)
@@ -1340,31 +1536,40 @@ def get_average_background_recangle(port,image):
         print(E)
         return 0
 
-valid_FOV_circle = {
-    10: [562, 364, 550],#, 600],
-    11: [502, 404, 520],#, 570]
-    20: [535, 374, 510],
-    21: [548, 404, 505],
-    30: [517, 389, 520],
-    31: [562, 364, 520],
-    40: [542, 394, 520],
-    41: [522, 394, 520],
-    50: [640, 454, 620],
-    51: [512, 379, 510]
-    }
+def get_average_background_recangle(port,image):
+    try:
+        points=valid_background_rectangle[port]
+#        FOV=get_FOV_mask(port)
+        dummy=image[points[1]:points[3],points[0]:points[2]]#FOV*image#
+        return np.min(dummy[np.nonzero(dummy)])
+    except Exception as E:
+        print(E)
+        return 0
+
+def estimate_offset(port,program,plot_it=False):
+    explist=[1,2,3,4,5,6,7,8,9]
+    meanlist=[]
+    exlist=[]
+    FOV=get_FOV_mask(port)
+    for exposure in explist:        
+        exist,time,frame=download_background_by_program(port,program,exposure)
+        if exist:
+            meanlist.append(np.mean(frame[np.where(FOV==1)]))
+            exlist.append(exposure)
+    if len(meanlist)>=2:
+#        print(exlist,meanlist)
+        from scipy.stats import linregress
+        slope, intercept, r_value, p_value, std_err = linregress(exlist,meanlist)
+        if plot_it:
+            plt.figure()
+            plt.plot(exlist,meanlist,'b*')
+            plt.plot(exlist,np.array(exlist)*slope+intercept,'k-')
+            plt.show()
+            print("slope, intercept, r_value, p_value, std_err",slope, intercept, r_value, p_value, std_err)
+        return intercept
+    else:
+        return -1
 
-#valid_background_rectangle = {
-#    10: [300,0 ,800 ,200],
-#    11: [200,100,700,200],
-#    20: [100,100,400,200],
-#    21: [150,0,800,200],
-#    30: [200,0,800,200],
-#31: [200,100,700,200],
-#40: [200,100,700,200],
-#41: [200,100,700,200],
-#50: [200,100,700,200],
-#51: [200,100,700,200]
-#    }
     
 if __name__=='__main__':
     print("local function calling")
@@ -1372,60 +1577,122 @@ if __name__=='__main__':
 #    #status,time,images,valid=get_temp_from_raw_by_program_V1(51,"20171018.019",time_s=0,emi=0.8,divertorpart="all",version=1)
 ##    status1,time1,images1,valid1=get_temp_from_raw_by_program_V1(41,"20171024.016",time_s=[0.75,0.99],emi=0.8,version=1)
 ##    exist,time1,frames=download_raw_images_by_program_via_png(41,"20171024.016",time_s=[0.75,0.99],version=1,threads=1)
-    status,time,images,valid=get_temp_from_raw_by_program(20,"20171129.036",time_s=4.5,emi=0.8,T_version=1,version=1,threads=4,give_ERROR=False)
-    status2,time2,images2,valid2=get_temp_from_raw_by_program(20,"20171129.036",time_s=4.5,emi=0.8,T_version=2,version=1,threads=4,give_ERROR=False)    
-    max1=[]
-    max2=[]
-    ave1=[]    
-    ave2=[]
-    for t in range(len(time)):
-        bla1=images[t][380:400,500:540]
-        bla2=images2[t][380:400,500:540]
-        max1.append(np.max(bla1))
-        ave1.append(np.average(bla1))
-        max2.append(np.max(bla2))
-        ave2.append(np.average(bla2))
-    plt.figure()
-    plt.plot(time,max1,label="max V1")        
-    plt.plot(time,max2,label="max V2")        
-    plt.plot(time,ave1,label="average V1")        
-    plt.plot(time,ave2,label="average V2")
-    plt.legend()
-    plt.savefig('20171129.036 - T calibration comparison.png', dpi=300, bbox_inches='tight')
-#    background1,LUT1,refT1,gain1,offset1,badpixels1,t_exp1,cfilter1=get_calib_data(41,"20171024.016",0.8,1,version=1)
-##    background2,LUT2,refT2,gain2,offset2,badpixels2,t_exp2,cfilter2=get_calib_data(51,"20171024.016",0.8,2,version=1)
-#    prog=AKF_1.get_program_from_PID("20171024.015")
-#   apply NUCs=[]
-#    gains=[]
-#    offsets=[]
-#    mo=[]
-#    mg=[]
-#    if prog[0]:
-#        t0=prog[1]['trigger']['0'][0]
-#        t1=prog[1]['trigger']['1'][0]
-#        t6=prog[1]['trigger']['6'][0]
-#        for i in range(4,10):
-#            NUC_DL=download_NUC_by_times(41,t0,t1,i,1)
-#            NUCs.append(NUC_DL)
-#            offsets.append(NUC_DL[1][1])
-#            mo.append(np.median(offsets[i-4]))
-#            gains.append(NUC_DL[1][0])
-#            mg.append(np.median(gains[i-4]))
-#    plt.figure()
-#    plt.plot([4,5,6,7,8,9],mg,label="gain")
-#    plt.figure()
-#    plt.plot([4,5,6,7,8,9],mo,label="offets")
-#    plt.figure()
-#    plt.imshow(images1[-1],vmin=45+273.15,vmax=500+273.15)
-#    plt.figure()
-#    plt.imshow(images2[-1],vmin=45+273.15,vmax=500+273.15)
-##    plt.figure()
-##    plt.imshow(badpixels1,cmap='gray')
-#    plt.figure()
-#    plt.imshow(frames[16],vmin=np.average(frames[16])-500,vmax=np.average(frames[16])+1500)
+#    import h5py as h5
+#    
+#    for prog in ["20171108.015"]:#"20171109.010","20170926.009","20171207.009"]:
+#        for port in [20,21]:
+#            try:
+#                texp=get_exposure_by_program(port,prog)
+#            except:
+#                texp=[False]
+#            if texp[0]:
+#                texp=texp[2][0]
+#                background=download_background_by_program(port,prog,texp)
+#                
+#                if background[0]:
+#                    
+##                    btest=check_backgroundframe(background[2])
+##                    if not btest[0]:
+#                    plt.figure()
+#                    plt.imshow(background[2],vmin=np.mean(background[2])-200,vmax=np.mean(background[2])+200)
+##                        plt.colorbar()
+#                    plt.title(prog+" "+str(port))
+
+#                cold=download_NUC_by_program(port,prog,texp)
+#                if cold[0]:
+#                    if port==11:
+#                        portnr=21
+#                    elif port==21:
+#                        portnr=11
+#                    else:
+#                        portnr=port
+#                    camera=portcamdict['OP1.2a']['AEF'+str(portnr)]
+#                    path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Calibrations\\IRCamReferenceImages\\"
+##                    path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\v1.0.1\\QIR-IrAnalysis\\release\\Config\\Thermal calibration\\"
+#                    file=h5.File(path+"IRCAM_"+camera.split("_")[2]+"\\"+camera+"_nuc_reference_cold_"+str(int(texp))+"us.h5",'r')
+##                    file=h5.File(path+camera+"\\"+camera+"_nuc_reference_cold_"+str(int(texp))+"us.h5",'r')
+#                    refframe=np.array(file[camera+"_nuc_reference_cold_"+str(int(texp))+"us.h5"])
+##                    ctest=check_coldframe_by_refframe(cold[1][2],refframe,plot_it=False)
+#                    references=[]
+#                    shapi=np.shape(refframe)
+#                    for i in [int(shapi[1]//4),int(shapi[1]//2),int(shapi[1]//4*3)]:
+#                        dataline=refframe[0:shapi[0],i]    
+#                        references.append(dataline-np.mean(dataline))
+#                    ctest=IR_tools.check_coldframe(cold[1][2],references,plot_it=True)
+#                    print(ctest,port,prog)
+#                    if not ctest[0]:
+#                    bla=np.array(cold[1][2])-np.array(refframe)
+#                    print(np.mean(bla),np.max(bla),np.min(bla),np.std(bla),(np.max(bla)-np.min(bla))/np.mean(bla),port)
+#                    dataset=[]
+#                    threshold=100
+#                    shapi=np.shape(bla)
+#                    plt.figure()
+#                    for i in [int(shapi[1]//4),int(shapi[1]//2),int(shapi[1]//4*3)]:
+#                        referenceline=bla[0:shapi[0],i]    
+#                        meanref=referenceline-np.mean(referenceline)
+#                        dataset.append(meanref)#np.max(meanref[100:-99])-np.min(meanref[100:-99]))
+##                        plt.plot(meanref,label=str(i))
+#                    for j in range(len(dataset)):
+#                        if j==0:
+#                            plt.plot(dataset[j]-dataset[-1],label=str(j))
+#                        else:
+#                            plt.plot(dataset[j]-dataset[j-1],label=str(j))
+#                    plt.title(prog+"_AEF"+str(port))
+#                    if np.mean(dataset)>threshold:
+#                        valid=False    
+#                    else:
+#                        valid=True
+#                    print(valid,np.mean(dataset))#,np.mean(dataset[0]-dataset[-1]),np.mean(dataset[1]-dataset[0]),np.mean(dataset[2]-dataset[1]))
+#                    plt.figure()
+#                    plt.imshow(cold[1][2],vmin=np.mean(cold[1][2])-500,vmax=np.mean(cold[1][2])+500)
+#                    plt.colorbar()
+#                    plt.title(prog+" "+str(port))
+#                    plt.figure()
+#                    plt.imshow(bla,vmin=np.mean(bla)-100,vmax=np.mean(bla)+100)
+#                    plt.colorbar()
+#                    plt.figure()
+#                    plt.imshow(refframe,vmin=np.mean(refframe)-100,vmax=np.mean(refframe)+100)
+#                    plt.colorbar()
+#                    plt.title("referenceframe "+str(port))
+                        
+                        
+
+    port=20
+    prog="20171108.015"
+#    status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,0.2],emi=0.82,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=True)
+    bla=get_exposure_by_program(port,prog)
+    status2,time2,images2=download_raw_images_by_program_via_png(port,prog,time_s=[0,0.2],threads=1)
 #    plt.figure()
-#    plt.imshow(gain2,vmin=0,vmax=10)
+#    plt.imshow(images[0],vmin=300,vmax=400)
+#    plt.colorbar()
+#    tmax=[]
+#    port=20
+    NUCblock=download_NUC_by_program(port,prog,bla[2][0])
+    background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(port,prog)
+    refcoldhot=IR_tools.load_ref_images("AEF"+str(port),bla[2][0])
+    recal_gain_offset=IR_tools.calculate_gain_offset_image_pix(NUCblock[1][2].copy(),None,refcoldhot[0].copy(),refcoldhot[1].copy())
+    plt.figure()
+    plt.imshow(NUCblock[1][0]-recal_gain_offset[0])
+    plt.title("gain")
+    plt.figure()
+    plt.imshow(NUCblock[1][1]-recal_gain_offset[1])
+    plt.title("offset")
+#    ima1=apply_NUC(images2.copy(),NUCblock[1][0],NUCblock[1][1])
+#    ima2=apply_NUC(images2.copy(),recal_gain_offset[0],recal_gain_offset[1])
+    ima1=apply_calib_on_raw(images2.copy(),background,LUT,gain=NUCblock[1][0],offset=NUCblock[1][1])
+    ima2=apply_calib_on_raw(images2.copy(),background,LUT,gain=recal_gain_offset[0],offset=recal_gain_offset[1])
+    
+    plt.figure()
+    plt.imshow(ima1[0],vmin=300,vmax=400)
+    plt.colorbar()
+    plt.title("aleix nuc")
+    
+    plt.figure()
+    plt.imshow(ima2[0],vmin=300,vmax=400)
+    plt.colorbar()
+    plt.title("adnan nuc")
+#    for i in range(len(images)):
+#        tmax.append(np.max(images[i]))
 #    plt.figure()
-#    plt.imshow(offset2,vmin=np.average(offset2)-100,vmax=np.average(offset2)+100)
-
-    
\ No newline at end of file
+#    plt.plot((time-time[0])/1e9,np.array(tmax)-273.15)
+#            
diff --git a/uploadingversionIRdata.py b/uploadingversionIRdata.py
index d6b0c88a572206767612a3839f0639791e38508d..8f71343f8ce87371a9bfa38471f31690d0e7ce14 100644
--- a/uploadingversionIRdata.py
+++ b/uploadingversionIRdata.py
@@ -576,7 +576,7 @@ def upload_NUC_ref(camera,portnum,time,texp_filter,gain,offset,cold,bad,gain_e=[
     
     upload_fulldataset(url,parobj,dataobj,version,parlog,data)
 
-def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason=""):
+def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason="",extra_par=None):
     """
     upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason="")
     (numpy.array)   LUTs:       t x n x 3 array DL,T,T_er, in general t=1, n= number of look-up-tables
@@ -615,6 +615,8 @@ def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason=""
     else:
         print("multiple timestamps for LUTs are not jet implemented")
         exit()
+    if type(extra_par)==dict:
+        structure.update(extra_par)
     url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
     if newversion:        
         archive_create_stream_version(url, reason, False, "n/a", "n/a" ) 
@@ -643,25 +645,61 @@ def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason=""
             
     upload_fulldataset(url,parobj,dataobj,version)
 
-#def upload_NUC_ref():
-#    """
-#    
-#    """
-#    print("not ready")
-
-def download_LUT_tables(camera,port,time,texp=0,emissivity=0,Filter=0):
-    """
-    (str)           camera:     string of cameraname "IRCAM"  or "INFRATEC", is used for the path
-    (numpy.array)   time:       time vector in uint64 ns of length t
-    (bool)          newversion: defines whether a new version is needed    
-    (int)           portnum:    portnumber as int, example: 10 or 11 etc.
-    in case of texp=0 and emissivity=0 it will give back all LUTs
+def upload_hot_cold_reference(port,path):
     """
     
+    """
+    if port==50:
+        camera="INFRATEC"
+    else:
+        camera="IRCAM"
+    group="QRT_"+camera#+"_TEST" # test part will be removed in final version
+    name=["raw"]
+    datastream_name="AEF"+str(port)+"_raw"  
     print("not ready")
-    return 0
-    
-
+   
+def upload_scene_model(port,time_ns,path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Calibrations\\scene_models",newversion=False,reason=""):
+    if port==50:
+        camera="INFRATEC"
+    else:
+        camera="IRCAM"
+    group="QRT_"+camera#+"_TEST" # test part will be removed in final version
+    name=["scene_model"]
+    datastream_name="AEF"+str(port)+"_scene_model"    
+    url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name    
+    Fullpath=os.path.join(path,"AEF"+str(port),"AEF"+str(port)+"_scene_model.h5")
+    File=h5reader.File(Fullpath,'r')
+    keylist=list(File.keys())
+    time_ns_list=[]
+    framelist=[]
+    ii=0
+    if newversion:        
+        archive_create_stream_version(url, reason, False, "n/a", "n/a" ) 
+        ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
+        version=ver
+    else:
+        ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
+        if ver==None:
+            reason="Versioning of Data"
+            archive_create_stream_version(url, reason, False, "n/a", "n/a" )
+            version=1
+        else:
+            version=ver
+    structure={}
+    for key in keylist:
+        framelist.append(np.array(File[key]))
+        time_ns_list.append(time_ns+20*ii)
+        ii+=0
+        structure[key]=time_ns+20*ii
+        
+    chanDesc={"[0]":{'name':name[0],'physicalQuantity':{'type':'none'},'active':1}}
+    parobj={"label":"parms",
+            "description":"scene model for AEF"+str(port),
+            "values":[{"chanDescs":chanDesc,"structure":structure}],
+            "dimensions":[time_ns_list[0],time_ns_list[-1]]
+            }    
+    upload_Parlog(url,parobj,version=version)  
+    upload_images(camera,port,framelist,time_ns_list,"scene_model",version)
     
 def upload_Parlog(url,parobj,versionnumber=0):
 #    print('>upload_Parlog: json of parlog')
@@ -696,7 +734,7 @@ def upload_fulldataset(url,parlog,data,versionnumber,bparlog=True,bdata=True):
     
 ### from here on subfunctions    
     
-def upload_images(camera,portnum,images,time,typ,newversion=False,reason=""):
+def upload_images(camera,portnum,images,time,typ,version):
     """
     upload_images(camera,portnum,images,time,metadata={},newversion=False,reason="")
     
@@ -707,13 +745,13 @@ def upload_images(camera,portnum,images,time,typ,newversion=False,reason=""):
     datastream_name="AEF"+str(portnum)+"_"+typ#+"_TEST001"
     
     #url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
-    if newversion:
-        print('no version control of HDF5 files yet!')
+#    if newversion:
+#        print('no version control of HDF5 files yet!')
 #        archive_create_stream_version(url, reason, False, "n/a", "n/a" )     
     try:
 #        archive.write_signals(images,time,group,project,names, datastream_name,units,metadata,2,True)
         tmp=writeH5(datastream_name,images,time)
-        uploadH5(group,datastream_name,tmp,True)
+        uploadH5_Versioning(group,datastream_name,tmp,version,True)
     except Exception as E:
         print("Upload Failed")
         print(E)