add function in IR_image_tools, bug fixes for OP1.2b upload

IR_image_tools.py

@@ -88,6 +88,14 @@ def quad_abweich(data,fit):
         print("Arrays must have same dimensions")
     return R    
         
+def find_nearest(array,value):
+    #a=array
+    a = [x - value for x in array]
+    mini = np.min(np.abs(a))
+    try: idx= a.index(mini)
+    except: idx= a.index(-mini)
+    return idx#array[idx]
+        
 def check_coldframe(coldframe,references=None,threshold=0.5,plot_it=False):
     '''
     return true/false and the quality factor

Upload_Day_IRdata.py

@@ -119,7 +119,7 @@ fout=open("upload_log_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minu
 #foutf=open("upload_log_failes_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
 sys.stdout = Tee(sys.stdout, fout)
 #sys.stdout = fout
-
+f_success=open("upload_log_successlist_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
 #%% decide data path format
 for date in dates:
     if date[0]==2017 and date[1]<=9 and date[2]<=25:
@@ -153,6 +153,8 @@ for date in dates:
         port = int(ports[i].split('AEF')[1])    
         path = rawdatapath.format(pc_list[i],date)
         print(datetime.datetime.now(),"Starting Upload for "+ports[i],date)
+        f_success.write(ports[i]+" \n")
+        f_success.write("program folder meta1 meta2 raw1 raw2 \n")
 #        foutf.write(datetime.datetime.now(),"Starting Upload for "+ports[i],date,"\n")
         # find all programs of this date on this PC
         programtimes = []
@@ -245,7 +247,7 @@ for date in dates:
                                         except Exception as E:
                                             print("upload of NUC files for "+str(t_exp)+"us failed")
                                             print(E) 
-                                            foutf.write(datetime.datetime.now(),"upload of NUC files for "+str(t_exp)+"us failed",E,"\n")
+#                                            foutf.write(datetime.datetime.now(),"upload of NUC files for "+str(t_exp)+"us failed",E,"\n")
                                     else:
                                         print("nuc upload deactivated")
 #                                        foutf.write(datetime.datetime.now(),"nuc upload deactivated","\n")
@@ -399,6 +401,24 @@ for date in dates:
                                     frametype2.append(-1)
                         except:
                             frametype2=np.zeros((len(framenumber2)))-1
+                        wrong1,correction_list1=IRup.check_timevector_framenumbers(time1,framenumber1)
+                        wrong2,correction_list2=IRup.check_timevector_framenumbers(time2,framenumber2)
+                        time1_0=time1.copy()
+                        time2_0=time2.copy()
+                        if wrong1:
+                            sensor1=np.delete(sensor1,correction_list1)
+                            chip1=np.delete(chip1,correction_list1)
+                            framerate1=np.delete(framerate1,correction_list1)
+                            frametype1=np.delete(frametype1,correction_list1)
+                            expo1=np.delete(expo1,correction_list1)                            
+                            time1=np.delete(time1,correction_list1)
+                        if wrong2:
+                            sensor2=np.delete(sensor2,correction_list2)
+                            chip2=np.delete(chip2,correction_list2)
+                            framerate2=np.delete(framerate2,correction_list2)
+                            frametype2=np.delete(frametype2,correction_list2)
+                            expo2=np.delete(expo2,correction_list2)                            
+                            time2=np.delete(time2,correction_list1)
                         channel_names=["sensortemperature","cameratemperature","framerate","exposuretime","framenumber","frametype"]
                         channel_units=["oC","oC",'Hz','us','none','none']
                         channel_values1=[sensor1, chip1,framerate1,expo1,framenumber1,frametype1]
@@ -415,8 +435,10 @@ for date in dates:
                             toff=0
                         print(datetime.datetime.now(),"   regenerate timevectors")
                         time1=IRup.regenerate_timevector(time1,0,t2,0)
-                        time2=IRup.regenerate_timevector(time2,1,t2,toff)
+                        time2,Program=IRup.regenerate_timevector(time2,1,t2,toff,True)
                     else:
+                        correction_list1=[]
+                        correction_list2=[]
                         channel_names=["sensortemperature","cameratemperature","framerate","exposuretime"]
                         channel_units=["oC","oC",'Hz','us']
                         channel_values1=[sensor1, chip1,framerate1,expo1]
@@ -425,7 +447,9 @@ for date in dates:
                         comment_meta2=""
                         print(datetime.datetime.now(),"   regenerate timevectors")
                         time1=IRup.regenerate_timevector(time1,0,t2,0)
-                        time2=IRup.regenerate_timevector(time2,[1,6],t2,0)
+                        time2,Program=IRup.regenerate_timevector(time2,[1,6],t2,0,True)
+                        time1_0=time1
+                        time2_0=time2
                     hf1.close()
                     del attri1
                     hf2.close()            
@@ -434,19 +458,31 @@ for date in dates:
                     print(datetime.datetime.now(),"   Upload Meta Stream(s)")
                     if metastream:
                         try:
-                            IRup.upload_meta_stream_V2(cameras[i],port,np.asarray(time1),channel_names,channel_units,channel_values1,newversion=newversion,reason=reason)
+                            meta1up=IRup.upload_meta_stream_V2(cameras[i],port,np.asarray(time1),channel_names,channel_units,channel_values1,newversion=newversion,reason=reason, Return=True)
+                            if meta1up:
+                                metares1=1
+                            else:
+                                metares1=0
                         except Exception as E:
                             print(E)
+                            metares1=0
 #                            foutf.write(datetime.datetime.now(),"metastream 1 upload failed",E,"\n")
                         del sensor1, chip1,framerate1,expo1
                         try:
-                            IRup.upload_meta_stream_V2(cameras[i],port,np.asarray(time2),channel_names,channel_units,channel_values2,newversion=newversion,reason=reason)
+                            meta2up=IRup.upload_meta_stream_V2(cameras[i],port,np.asarray(time2),channel_names,channel_units,channel_values2,newversion=newversion,reason=reason, Return=True)
+                            if meta2up:
+                                metares2=1
+                            else:
+                                metares2=0
                         except Exception as E:
                             print(E)  
+                            metares2=0
 #                            foutf.write(datetime.datetime.now(),"metastream 2 upload failed",E,"\n")
                         del sensor2,chip2,framerate2,expo2
                     else:
                         print("metastream upload deactivated")
+                        metares1=0
+                        metares2=0
 #                        foutf.write(datetime.datetime.now(),"metastream upload deactivated","\n")
                     if raw_parlog or raw_data:                    
                         print(datetime.datetime.now(),"   Start Raw Data Upload for "+str(prog_time))
@@ -455,38 +491,49 @@ for date in dates:
                             trys=0
                             while (trys<retry and not success):
                                 try:
-                                    response=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time1,dtype=np.int64),path+"\\"+rlist[fkeys[f1]],metadata1,parlog=raw_parlog,rawdata=raw_data,newversion=newversion,reason=reason)#upload cold and backgrounds
+                                    response=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time1_0,dtype=np.int64),path+"\\"+rlist[fkeys[f1]],metadata1,parlog=raw_parlog,rawdata=raw_data,newversion=newversion,reason=reason,del_list=correction_list1)#upload cold and backgrounds
                                     if response[0]:
                                         success=True
+                                        rawres1=1
                                     else:
                                         print(datetime.datetime.now(),"raw 1 upload failed",response[1],response[2])
+                                        rawres1=0
 #                                        foutf.write(datetime.datetime.now(),"raw 1 upload failed",response[1],response[2],"\n")
                                 except Exception as E:
                                     trys+=1
                                     print("Upload for rawfile 1 failed!, try:"+str(trys),E)
+                                    rawres1=0
 #                                    foutf.write(datetime.datetime.now(),"Upload for rawfile 1 failed!, try:"+str(trys),E,"\n")
                                     
-                                
+                        else:
+                            rawres1=0
                         if raw2:
                             success=False
                             trys=0
                             while (trys<retry and not success):
                                 print(datetime.datetime.now(),"      Upload second file")
                                 try:
-                                    response=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time2,dtype=np.int64),path+"\\"+rlist[fkeys[f2]],metadata2,parlog=raw_parlog,rawdata=raw_data)#upload raw images from plasma
+                                    response=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time2_0,dtype=np.int64),path+"\\"+rlist[fkeys[f2]],metadata2,parlog=raw_parlog,rawdata=raw_data,del_list=correction_list2)#upload raw images from plasma
                                     if response[0]:
                                         success=True
+                                        rawres2=1
                                     else:
                                         print(datetime.datetime.now(),"raw 2 upload failed",response[1],response[2])
+                                        rawres2=0
 #                                        foutf.write(datetime.datetime.now(),"raw 2 upload failed",response[1],response[2],"\n")
                                 except Exception as E:
                                     trys+=1
                                     print("Upload for rawfile 2 failed!, try"+str(trys),E)
+                                    rawres2=0
 #                                    foutf.write(datetime.datetime.now(),"Upload for rawfile 2 failed!, try:"+str(trys),E,"\n")
-                                    
+                        else:
+                            rawres2=0
                         print(datetime.datetime.now(),"      Upload finished")
                     else:
                         print("raw upload deactivated")
+                        rawres1=0
+                        rawres2=0
+                    f_success.write(Program+" "+str(prog_time)+" "+str(metares1)+" "+str(metares2)+" "+str(rawres1)+" "+str(rawres2)+" \n")
 #                        foutf.write(datetime.datetime.now(),"raw upload deactivated","\n")
                 else:#folder is empty
                     print("no files in",prog_time)
@@ -530,5 +577,6 @@ for date in dates:
     ##"{:02d}".format(zahl)
     
 fout.close()
+f_success.close()
 #foutf.close()
 sys.stdout = original
\ No newline at end of file

downloadversionIRdata.py

@@ -186,6 +186,29 @@ def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, ve
         print("Warning: unable to find LUTs, check your request")
         return False,0
 
+
+def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
+    """
+    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
+    
+    Read LUT V3 from local files for testing new calibration.
+    """
+    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
+    OP = get_OP_by_time(this_time)
+    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
+    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
+#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
+    try:
+        with open(LUT_dir+"\\"+filename) as data_file:  
+            jfile=json.load(data_file)
+        LUT=np.array(jfile['LUT']).swapaxes(0,1)
+        if verbose>0:
+            print("succesfully loaded V3 LUT from local directory")
+        return True, LUT
+    except Exception as E:
+        print("ERROR in loading V3 LUTs",E)
+        return False, []
+    
 def download_NUC_by_program(port,program,exposure,version=0):
     prog=AKF_1.get_program_from_PID(program)
 #    try:
@@ -998,32 +1021,35 @@ def get_temp_from_raw_by_program_V1(portnr,program,time_s=0,emi=0.8,divertorpart
 def get_temp_from_raw_by_program_V2(portnr,program,time_s=0,emi=0.8,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False):
     return get_temp_from_raw_by_program(portnr,program,time_s=time_s,emi=emi,T_version=2,version=version,threads=threads,give_ERROR=give_ERROR,use_firstframe_as_background=use_firstframe_as_background)
   
-def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,back_emissivity=0.8):
-    background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version,back_emissivity)
-    print(datetime.datetime.now(),"raw download start")
+def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,back_emissivity=0.8,verbose=0):
+    background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version,back_emissivity,verbose=verbose-1)
+    if verbose>0:
+        print(datetime.datetime.now(),"raw download start")
     if fastDL: 
-        exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_s,version,threads)
+        exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_s,version,threads,verbose=verbose-1)
     else:
-        exist,time,frames=download_raw_images_by_program(portnr,program,time_s,version)
+        exist,time,frames=download_raw_images_by_program(portnr,program,time_s,version,verbose=verbose-1)
         if exist:
             frames=[im.swapaxes(0,1) for im in frames]
         else:
             raise Exception("no data found")
-    print(datetime.datetime.now(),"raw download finished")
+    if verbose>0:
+        print(datetime.datetime.now(),"raw download finished")
     FOV=get_FOV_mask(portnr)
     if use_firstframe_as_background:
         bim=apply_NUC([frames[0]],gain,offset)
         background=get_average_background_recangle(portnr,bim[0])
     if give_ERROR:
-        frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
+        frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
     else:
-        frames=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
-    
+        frames=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
     for i in range(len(frames)):
         frames[i]=(frames[i]*FOV).astype(np.float32)
-    print(datetime.datetime.now(),"correcting bad pixels")
+    if verbose>0:
+        print(datetime.datetime.now(),"correcting bad pixels")
     frames=IR_tools.correct_images(frames,badpixels)
-    print(datetime.datetime.now(),"checking temperaturerange")
+    if verbose>0:
+        print(datetime.datetime.now(),"checking temperaturerange")
     valid=True
 #    for i in range(len(frames)):
     try:
@@ -1331,7 +1357,7 @@ def apply_NUC(images,gain,offset,gain_error=0,offset_error=0,give_error=False):
         print(E)
         return 0
         
-def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivity=0.8, verbose=0):
+def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivity=0.82, verbose=0):
     prog=AKF_1.get_program_from_PID(program)
 #    try:
 #        t_program = AKF_2.get_program_from_to(program)
@@ -1389,10 +1415,27 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
                     refT=28.5
                 elif Temp_V==3:
                     if verbose>0:
-                        print('use temperature calibration version 3')
+                        print('use temperature calibration version 3, TEST CASE!!!')
+                    print('use temperature calibration version 3, TEST CASE!!!')
                     frame=background.copy()
                     background=get_average_background_recangle(port,background)
                     refT=28.5
+                    # read LUT V3 (compensated BG signal and photons flux of BG)
+                    LUT_DL_wall = read_LUT_from_file(port, time, t_exp, 0.45, cfilter, verbose=verbose-1)
+                    if LUT_DL_wall[0]:
+                        if verbose>0:
+                            print('got the wall LUT')
+                        LUT_wall = LUT_DL_wall[1]
+                        index=IR_tools.find_nearest(LUT_wall[1],refT)
+                        background=background-LUT_wall[0][index]
+                        refT=0
+                        del LUT_DL_wall,LUT_wall
+                    LUT_DL = read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
+                    if LUT_DL[0]:
+                        if verbose>0:
+                            print('got the V3 LUT')
+                        LUT = LUT_DL[1]
+                        del LUT_DL
 #                        back_off=estimate_offset(port,program)                      
 #                        background=(background-back_off)/(back_emissivity)+back_off
                 else:
@@ -1622,9 +1665,10 @@ if __name__=='__main__':
 #    prog="20171012.035"
     prog="20171005.024"
     prog="20171109.056"
+    prog="20171207.050"
     program="20180712.027"
-#    status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,6],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False)
-#    status,time,images=download_raw_images_by_program_via_png(port,prog,time_s=6,threads=4,verbose=10)
+    status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,1],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False)
+    status,time,images=download_raw_images_by_program_via_png(port,prog,time_s=1,threads=4,verbose=0)
 #    print(len(images))
 #    import h5py as h5
 #    szene=h5.File("X:\\E4 Diagnostics\\QIR\Calibrations\\scene_models\\AEF"+str(port)+"\\AEF"+str(port)+"_scene_model.h5")
@@ -1638,52 +1682,52 @@ if __name__=='__main__':
 #    cb=plt.colorbar()
 #    cb.set_label("temperature K", rotation=270,labelpad=26,fontsize=26)
 #    cb.ax.tick_params(labelsize=26)
-#    plt.figure()# #    plt.imshow(CAD,cmap='gray')
-#    plt.imshow(images[-1],vmin=300,vmax=600,cmap=exJet)
-#    cb=plt.colorbar()
-#    cb.set_label("temperature K", rotation=270,labelpad=26,fontsize=26)
-#    cb.ax.tick_params(labelsize=26)
-    version=0
-    time_s=30
-    prog=AKF_1.get_program_from_PID(program)
-    if prog[0]:
-        t0time=prog[1]['trigger']['0'][0]
-        starttime=prog[1]['trigger']['1'][0]
-        stoptime=prog[1]['trigger']['6'][0]
-        success=True
-        OP=get_OP_by_time(starttime)
-        Cam=portcamdict[OP]['AEF'+str(port)]
-        if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
-            if version==0:
-                version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
-            larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
-        elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
-            if version==0:
-                version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
-            larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
-        else:
-            raise Exception("Port number does not fit the known cameras")
-        if type(time_s)==list:
-            tstart=time_s[0]
-            tstop=time_s[1]
-            if tstop<tstart:
-                raise Exception("endtime before starttime")
-            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
-            stdate=stdate.isoformat()
-            enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
-            enddate=enddate.isoformat()
-        else:
-            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
-            stdate=stdate.isoformat()
-            t0date=datetime.datetime.utcfromtimestamp((t0time-100)/1e9)
-            t0date=t0date.isoformat()
-            if time_s==0:
-                enddate=datetime.datetime.utcfromtimestamp(stoptime+100/1e9)        
-                enddate=enddate.isoformat()
-            else:
-                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)  
-                enddate=enddate.isoformat()
-            #"2017-11-15 08:00:00"
-        times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
-        timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),stdate.replace("T"," "))#
-        print(len(times),len(timest0))
\ No newline at end of file
+    plt.figure()# #    plt.imshow(CAD,cmap='gray')
+    plt.imshow(images[-1],vmin=300,vmax=600,cmap=exJet)
+    cb=plt.colorbar()
+    cb.set_label("temperature K", rotation=270,labelpad=26,fontsize=26)
+    cb.ax.tick_params(labelsize=26)
+#    version=0
+#    time_s=30
+#    prog=AKF_1.get_program_from_PID(program)
+#    if prog[0]:
+#        t0time=prog[1]['trigger']['0'][0]
+#        starttime=prog[1]['trigger']['1'][0]
+#        stoptime=prog[1]['trigger']['6'][0]
+#        success=True
+#        OP=get_OP_by_time(starttime)
+#        Cam=portcamdict[OP]['AEF'+str(port)]
+#        if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+#            if version==0:
+#                version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
+#            larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+#        elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+#            if version==0:
+#                version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
+#            larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+#        else:
+#            raise Exception("Port number does not fit the known cameras")
+#        if type(time_s)==list:
+#            tstart=time_s[0]
+#            tstop=time_s[1]
+#            if tstop<tstart:
+#                raise Exception("endtime before starttime")
+#            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
+#            stdate=stdate.isoformat()
+#            enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
+#            enddate=enddate.isoformat()
+#        else:
+#            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
+#            stdate=stdate.isoformat()
+#            t0date=datetime.datetime.utcfromtimestamp((t0time-100)/1e9)
+#            t0date=t0date.isoformat()
+#            if time_s==0:
+#                enddate=datetime.datetime.utcfromtimestamp(stoptime+100/1e9)        
+#                enddate=enddate.isoformat()
+#            else:
+#                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)  
+#                enddate=enddate.isoformat()
+#            #"2017-11-15 08:00:00"
+#        times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
+#        timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),stdate.replace("T"," "))#
+#        print(len(times),len(timest0))
\ No newline at end of file

uploadingversionIRdata.py

@@ -325,7 +325,7 @@ def upload_meta_stream(camera,portnum,time,sensortemp,cameratemp,framerate,expos
             }
     upload_fulldataset(url,parobj,dataobj,version)
 
-def upload_meta_stream_V2(camera,portnum,time,channel_names,channel_units,channel_values,comment="",newversion=False,reason=""):
+def upload_meta_stream_V2(camera,portnum,time,channel_names,channel_units,channel_values,comment="",newversion=False,reason="",Return=False):
     """    
     old: sensortemp,cameratemp,framerate,exposuretime
     new: channels with names, units and values
@@ -362,7 +362,8 @@ def upload_meta_stream_V2(camera,portnum,time,channel_names,channel_units,channe
                  "values":channels.tolist(),
                  "dimensions":time.tolist()
                 }
-        upload_fulldataset(url,parobj,dataobj,version)
+        if Return: return upload_fulldataset(url,parobj,dataobj,version,Return=Return)
+        else: upload_fulldataset(url,parobj,dataobj,version)
     
 def upload_raw_data(camera,portnum,time,images,metadata={},comment="",newversion=False,reason=""):
     """
@@ -404,7 +405,7 @@ def upload_raw_data(camera,portnum,time,images,metadata={},comment="",newversion
     upload_Parlog(url,parobj,version)  
     upload_images(camera,portnum,images,time,"raw",newversion,reason)
     
-def regenerate_timevector(time,reftrig=1,tref=0,toff=0):
+def regenerate_timevector(time,reftrig=1,tref=0,toff=0,give_program=False):
     """
     regenerate_timevector(time,reftrig,tref,toff)
     (np.array)  time:   timevector of length t, 1d, in ns
@@ -423,13 +424,17 @@ def regenerate_timevector(time,reftrig=1,tref=0,toff=0):
             tref=prog[1].get('trigger').get(str(reftrig[0]))[0]
             tref2=prog[1].get('trigger').get(str(reftrig[1]))[0]
             ntime=np.asarray(match_time_vectors_lowlevel(time,tref+toff*1e6,tref2))
-        return ntime
+        if give_program:
+            return ntime,prog[1]['id']
+        else:
+            return ntime
     else:
         print("Program for this reference timestamp not found, cannot generate new timevector")
-        return np.asarray(time)
+        if give_program:
+            return np.asarray(time),"20000000.000"
 
     
-def upload_raw_data_from_H5(camera,portnum,time,h5file,metadata={},comment="",parlog=True,rawdata=True,newversion=False,reason=""):
+def upload_raw_data_from_H5(camera,portnum,time,h5file,metadata={},comment="",parlog=True,rawdata=True,newversion=False,reason="",del_list=[]):
     """
     upload_raw_data(camera,portnum,images,time,metadata={},newversion=False,reason="")
     (str)           camera:     string of cameraname "IRCAM"  or "INFRATEC", is used for the path
@@ -468,19 +473,21 @@ def upload_raw_data_from_H5(camera,portnum,time,h5file,metadata={},comment="",pa
         upload_Parlog(url,parobj,version)
     if rawdata:
         print("writing done, creating upload H5file")
-        tmp=writeH5_from_File(datastream_name,h5file,'images',time)
+        tmp=writeH5_from_File(datastream_name,h5file,'images',time,double_list=del_list)
         print("uploading H5file")
         if type(tmp) is list:
             n=1
             for ele in tmp:
+                success=True
                 print("uploading part "+str(n)+" of "+str(len(tmp)))
                 n+=1
                 bla= uploadH5_Versioning(group,datastream_name,ele,version,True)
                 if bla[0]:
                     print("upload of part "+str(n-1)+" of "+str(len(tmp))+" was successfull")
                 else:
+                    success=False
                     print("upload of part "+str(n-1)+" of "+str(len(tmp))+" failed")
-            return bla
+            return [success]
         else:
             return uploadH5_Versioning(group,datastream_name,tmp,version,True)
     
@@ -693,19 +700,14 @@ def upload_hot_cold_reference(port,path):
     """
     
     """
-#    if port==50:
-#        camera="INFRATEC"
-#    else:
+    
     camera="IRCAM"
-    group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
+    group="QRT_IRCAM_TEST"#+camera#+"_TEST" # test part will be removed in final version
     name=["raw"]
     datastream_name="AEF"+str(port)+"_raw"  
     print("not ready")
    
 def upload_scene_model(port,time_ns,path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Calibrations\\scene_models",newversion=False,reason=""):
-#    if port==50:
-#        camera="INFRATEC"
-#    else:
     camera="IRCAM"
     group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
     name=["scene_model"]
@@ -766,7 +768,7 @@ def download_LUT_tables(camera,port,time,texp=0,emissivity=0,Filter=0):
     
 
     
-def upload_Parlog(url,parobj,versionnumber=0):
+def upload_Parlog(url,parobj,versionnumber=0,Return=False):
 #    print('>upload_Parlog: json of parlog')
 #    print(json.dumps(parobj).encode("utf-8"))
     if versionnumber==0:
@@ -775,27 +777,39 @@ def upload_Parlog(url,parobj,versionnumber=0):
         parlog_request = urllib.request.Request( url + "_PARLOG/V"+str(versionnumber), data = json.dumps(parobj).encode("utf-8"),headers = { "Content-type" : "application/json" } )
     try:
         parlog_response = urllib.request.urlopen(parlog_request)
+        if Return: 
+            return parlog_response.getcode()//100 is 2 
     except urllib.error.HTTPError as ex:
-        msg = ex.read()
-        raise RuntimeError(msg)  
+        msg = ex.read() 
+        if Return: False
+        else:
+            raise RuntimeError(msg)  
     
-def upload_data(url,dataobj,versionnumber=0):
+def upload_data(url,dataobj,versionnumber=0,Return=False):
     if versionnumber==0:
         datastream_request = urllib.request.Request( url + "_DATASTREAM",data=json.dumps(dataobj).encode("utf-8"), headers={ "Content-type":"application/json" } )
     else:
         datastream_request = urllib.request.Request( url + "_DATASTREAM/V"+str(versionnumber),data=json.dumps(dataobj).encode("utf-8"), headers={ "Content-type":"application/json" } )
     try:
         datastream_response = urllib.request.urlopen(datastream_request)
+        if Return: return datastream_response.getcode()//100 is 2 
     except urllib.error.HTTPError as ex:
         msg = ex.read()
         raise RuntimeError(msg)
         
-def upload_fulldataset(url,parlog,data,versionnumber,bparlog=True,bdata=True):
+def upload_fulldataset(url,parlog,data,versionnumber,bparlog=True,bdata=True,Return=False):
     if bparlog:
-        upload_Parlog(url,parlog,versionnumber)
+        if Return:
+            res=upload_Parlog(url,parlog,versionnumber,Return)
+        else:
+            upload_Parlog(url,parlog,versionnumber,Return)
     if bdata:
-        upload_data(url,data,versionnumber)
-    
+        if Return:
+            res1=upload_data(url,data,versionnumber,Return)
+        else:                
+            upload_data(url,data,versionnumber)
+    if bdata and bparlog and Return:
+        return (res and res1)
     
 ### from here on subfunctions    
     
@@ -915,7 +929,7 @@ def writeH5(stream,data,dimof,idx=None):
 #        g.create_dataset(dataname, data=data.tolist(), dtype=dtype,compression="gzip")
 #    return tmpfile
 
-def writeH5_from_File(stream,filename,key,dimof,idx=None):
+def writeH5_from_File(stream,filename,key,dimof,idx=None,double_list=[]):
     """
     path=Path, data=numpy.array, dimof=list of long
     data.shape = (width,height,time) or (width,height) if len(dim)==1
@@ -932,8 +946,21 @@ def writeH5_from_File(stream,filename,key,dimof,idx=None):
         tmpfile += ".h5"
         with h5reader.File(tmpfile, 'w') as f:
             g = f.create_group('data')  # requires [row,col,time]
-            g.create_dataset('timestamps', data=list(dimof), dtype='uint64',compression="gzip")
-            f5.copy(key,f['/data'],name=stream)
+            
+            if len(double_list)==0:
+                g.create_dataset('timestamps', data=list(dimof), dtype='uint64',compression="gzip")
+                f5.copy(key,f['/data'],name=stream)            
+            else:
+                print("removing double frame for the upload HDF5")
+                shapi=f5[key].shape
+                dummy=np.zeros((shapi[0],shapi[1],shapi[2]),dtype=f5[key].dtype)
+                f5[key].read_direct(dummy)
+                dummy=np.delete(dummy,double_list,2)
+                dimof=np.delete(dimof,double_list)#done already earlier
+                g.create_dataset('timestamps', data=list(dimof), dtype='uint64',compression="gzip")
+                dset=g.create_dataset(stream,shape=(shapi[0],shapi[1],shapi[2]-len(double_list)),