Merge branch 'master' of gitlab.mpcdf.mpg.de:holn/IR_data_access into peter

CHANGELOG

 <<<<<<< HEAD
-
+26.03.2019: Upadte to V3.2.0:
+	- scne model download possible
+	- scene model reference download possible for heat flux data
+	- downsampled temperature image stream possible
+	- extration of temperature profiles possible
 27/28/29.11.2018: update to V3
 	- add file upload_heatflux.py
 	- add plot_heatflux_example.py
@@ -44,6 +48,7 @@ bugfixes in downloadversionIRdata:
     - fixed: wrong time intervall for TC divertor data
 
 Versions:
+V3.2.0: download of scene models possible, temperature profiles can be extracted, downsampled temperature images available
 V3.1.0: add of divertor loads in the upload and the download
 V3.0.2: change the upload of heat flux from float64 to float32. extract profile accepts now also timewindows.
 V3.0.1: extension of the heat flux upload (scene model reference can be uploaded) 

IR_image_tools.py

@@ -70,6 +70,18 @@ def bestimmtheitsmass_general(data,fit):
     else:
         print("bestimmtheitsmass_general: Arrays must have same dimensions")
     return R
+
+def bestimmheitsmass_linear(data,fit,debugmode=False):
+    R2=0
+    if len(fit)==len(data):
+        mittel_D=np.mean(data)#np.sum(data)/len(data)
+        mittel_F=np.mean(fit)
+        R2=quad_abweich_mittel(fit,mittel_D)/quad_abweich_mittel(data,mittel_D)
+        if debugmode:
+            print(mittel_D,mittel_F,quad_abweich_mittel(fit,mittel_D),quad_abweich_mittel(data,mittel_D),R2)
+    else:
+        print("bestimmtheitsmass_linear: Arrays must have same dimensions")
+    return R2
     
 def quad_abweich_mittel(data,mittel):
     R=0

ToDO.txt

@@ -6,14 +6,3 @@ download:
 - implement download of the stored temperature data (After the upload)
 - implement download of the stored heat flux data --> done in V3.0.0
 - implement download of FLIR data --> Done in V2.8.0, in testing phase
-
-upload:
-- upload the new nucing files for AEF50 (the workaround)
-- upload remaining data from failed uploads
-- upload scene models
-- upload temperature 
-- upload heat flux data (started)
-- upload FLIR data (partly done)
-- upload FLIR LUT (partly done)
-
-

plot_heatflux_example.py

 # -*- coding: utf-8 -*-
 """
 Created on Thu Nov 29 17:41:40 2018
-V3.0.2
+V3.2.0
 @author: holn
 """
 import numpy as np
-import downloadversionIRdata as IR
+import IR_data_access.downloadversionIRdata as IR
 import matplotlib.pyplot as plt
 import datetime
+
+port=20
+program="20181004.012"
+
 if __name__=='__main__':
     
     #%% loading data
     print(datetime.datetime.now(),"start")
-    status,times,images=IR.download_heatflux_by_program(20,"20171109.008",timewindow=0.9,version=1,threads=1,verbose=5)
+    status,times,images=IR.download_heatflux_by_program(port,program,time_window=0.9,version=1,threads=1,verbose=5)
     print('done')
     
     #%% plotting data
-    status2,time,s,q=IR.extract_heatflux_profile_from_DL((np.asarray(times)-times[0])/1e9,images,profile="TM3h_5_5",timewindow=0.5,verbose=5)
-    plt.figure()
-    plt.imshow(images[-1]/1e6)
-    plt.clim([0,4])
-    plt.tick_params(axis='both',       # changes apply to the x-axis
-                    which='both',      # both major and minor ticks are affected
-                    bottom='off',      # ticks along the bottom edge are off
-                    top='off',         # ticks along the top edge are off
-                    left='off',
-                    right='off',
-                    labelleft='off',
-                    labelbottom='off') # labels along the bottom edge are off
-    c_ax = plt.colorbar()
-    c_ax.set_label('q [MW/m2]')
-    plt.savefig('20171109.008 - '+str(round((np.asarray(times[-1])-times[0])/1e9,2))+' - AEF20IR - in MWm2.png', dpi=300, bbox_inches='tight')
-    if len(np.shape(q))==1 or np.shape(q)[0]==1:
-        if np.shape(q)[0]==1:
-            q=q[0]
+    if status:
+        status2,time,s,q=IR.extract_heatflux_profile_from_DL((np.asarray(times)-times[0])/1e9,images,profile="TM3h_5_5",time_window=0.5,verbose=5)        
         plt.figure()
-        plt.plot(s,q/1e6,'ko-')
-        plt.xlabel("s [m]",fontsize=26)
-        plt.ylabel("q [MW/m2]",fontsize=26)
-        plt.tick_params(labelsize=26)
-        plt.title("20171109.008 heat flux profile for TM3, finger 5, profile 5 @1s",fontsize=26)
-        plt.grid()
-        plt.ylim(bottom=0)
-        plt.show()
-    elif len(np.shape(q))==2:
-        plt.figure()
-        X,Y=np.meshgrid(time,s)
-        plt.pcolor(X,Y,q.swapaxes(0,1)/1e6,vmin=0)
-        plt.xlabel("time [s]",fontsize=26)
-        plt.ylabel("s [m]",fontsize=26)
-        plt.tick_params(labelsize=26)
-        cb=plt.colorbar()
-        cb.set_label('heatflux in MW/m²', rotation=270, labelpad=15,fontsize=26)
-        cb.ax.tick_params(labelsize=26)
-        plt.title("20171109.008 heat flux profile for TM3, finger 5, profile 5",fontsize=26)
-        plt.show()
+        plt.imshow(images[-1]/1e6)
+        plt.clim([0,4])
+        plt.tick_params(axis='both',       # changes apply to the x-axis
+                        which='both',      # both major and minor ticks are affected
+                        bottom='off',      # ticks along the bottom edge are off
+                        top='off',         # ticks along the top edge are off
+                        left='off',
+                        right='off',
+                        labelleft='off',
+                        labelbottom='off') # labels along the bottom edge are off
+        c_ax = plt.colorbar()
+        c_ax.set_label('q [MW/m2]')
+        plt.savefig('20171109.008 - '+str(round((np.asarray(times[-1])-times[0])/1e9,2))+' - AEF20IR - in MWm2.png', dpi=300, bbox_inches='tight')
+        if len(np.shape(q))==1 or np.shape(q)[0]==1:
+            if np.shape(q)[0]==1:
+                q=q[0]
+            plt.figure()
+            plt.plot(s,q/1e6,'ko-')
+            plt.xlabel("s [m]",fontsize=26)
+            plt.ylabel("q [MW/m2]",fontsize=26)
+            plt.tick_params(labelsize=26)
+            plt.title("20171109.008 heat flux profile for TM3, finger 5, profile 5 @1s",fontsize=26)
+            plt.grid()
+            plt.ylim(bottom=0)
+            plt.show()
+        elif len(np.shape(q))==2:
+            plt.figure()
+            X,Y=np.meshgrid(time,s)
+            plt.pcolor(X,Y,q.swapaxes(0,1)/1e6,vmin=0)
+            plt.xlabel("time [s]",fontsize=26)
+            plt.ylabel("s [m]",fontsize=26)
+            plt.tick_params(labelsize=26)
+            cb=plt.colorbar()
+            cb.set_label('heatflux in MW/m²', rotation=270, labelpad=15,fontsize=26)
+            cb.ax.tick_params(labelsize=26)
+            plt.title("20171109.008 heat flux profile for TM3, finger 5, profile 5",fontsize=26)
+            plt.show()
+#        status3,mapping=IR.download_heatflux_scene_model_reference(port,program=program,verbose=0)
+#        bild=np.zeros((768,1024))
         
\ No newline at end of file

upload_heatflux.py

-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 21 09:32:29 2018
-V3.0.2
-@author: holn
-"""
-
-import numpy as np
-import scipy.io as sio
-#import matplotlib.pyplot as plt
-import h5py as h5
-import uploadingversionIRdata as IRup
-import downloadversionIRdata as IR
-import datetime
-#125,190 vertical, horizontal *3 mm
-
-
-
-port=10
-program="20171109.008"
-#programlist=["20171011.009"]
-programlist=["20180814.024","20181016.016"]#["20171108.018","20171109.040","20171109.045"]
-path="F:\\tmp\\upload_it\\"#"F:\\holn\\Documents\\Doktorarbeit\\RESULTS\\OP1.2a\\Fulldivertor_res\\EJM\\H\\"
-outpfad="F:\\holn\\Documents\\Doktorarbeit\\DATEN\\Mapping\\LinesforStacks\\"
-scenepath=""
-
-project="W7XAnalysis"
-group="QRT_IRCAM"
-stream="AEF"+str(port)+"_heatflux"    
-comment="Test upload"    
-header={'code_version':4,'alpha_corrected':1}
-stream2="Mapping_reference"
-upload_Mapping=False
-upload_scene_reference=False
-stream_scene="AEF"+str(port)+"_SceneModel_reference"    
-scene_reference_time=0#ns timestamp goes here!
-
-for program in programlist:
-
-    File=sio.loadmat(path+program+"_AEF"+str(port)+"_heatflux_V4.mat")
-    time=np.asarray(File['time'][0])
-    tilelist=np.asarray(File['tiles'])
-    ### camera, program dependent things###
-    heatpics=np.zeros((125+190+10,108*12+4,len(time)),dtype=np.float32)# define the pictures for the heat flux
-    alphapic=np.zeros((125+190+10,108*12+4,1),dtype=np.float32) # define the picture for the alpha values
-    alphapic.fill(np.nan)
-    heatpics.fill(np.nan)
-    if upload_Mapping:
-        ### mapping dependent things ###
-        Xpic=alphapic.copy() #X-Coordiate in the Mapping
-        Ypic=alphapic.copy() #Y-Coordiate in the Mapping
-        FXpic=alphapic.copy() #X-Coordiate in the Fingersystem in the Mapping
-        FYpic=alphapic.copy() #Y-Coordiate in the Fingersystem in the Mapping
-        profpic=alphapic.copy() #profiles or s coordiante in the Mapping
-    #    profnpic=alphapic.copy() 
-        tilepic=alphapic.copy() #target indentifier, 1: low iota, 2: middle part, 3: high iota, 4: vertical
-        fingerpic=alphapic.copy() #finger ID from the mapping, last two digits are the profile number, the 3 to 6. digit are the finger id (100*i+j; i:0-136; j:0-16)
-    if upload_scene_reference:
-        ### Camera ###
-        PXpic=alphapic.copy() #orignal Pixel X
-        PYpic=alphapic.copy() #orignal Pixel Y
-        Xpic3D=alphapic.copy() #CAD 3D X from Scene Model
-        Ypic3D=alphapic.copy() #CAD 3D Y from Scene Model
-        Zpic3D=alphapic.copy() #CAD 3D Z from Scene Model
-        CID=alphapic.copy() #Component ID from the Scene Model
-        szene=h5.File(scenepath+"AEF"+str(port)+"_scene_model_"+str(scene_reference_time)+".h5",'r')
-        Xloc=np.asarray(szene['x'])
-        Yloc=np.asarray(szene['y'])
-        Zloc=np.asarray(szene['z'])
-        #CAD=np.asarray(szene['CAD'])
-        PFC=np.asarray(szene['PFC'])
-    
-    linedict={'h_l':30,'h_m':54,'h_h':24,'v':29}
-    numstacks={            
-    'h_l':{0:13,1:14,2:14,3:14,4:14,5:14,6:14,7:14,8:14,9:14,10:15,11:15,12:14,13:14,14:14,15:13,16:13,17:13,18:13,19:13,20:13,21:13,22:13,23:13,24:13,25:13,26:13,27:13,28:13,29:13},
-    'h_m':{0:14,1:15,2:14,3:15,4:14,5:15,6:14,7:15,8:14,9:15,10:14,11:15,12:14,13:15,14:14,15:15,16:14,17:15,18:14,19:15,20:14,21:15,22:14,23:15,24:14,25:15,26:14,27:15,28:14,29:15,30:14,31:15,32:14,33:15,34:14,35:15,36:14,37:15,38:14,39:15,40:14,41:15,42:14,43:15,44:14,45:15,46:14,47:15,48:15,49:14,50:14,51:14,52:14,53:15},
-    'h_h':{0:16,1:15,2:15,3:15,4:15,5:15,6:15,7:15,8:15,9:15,10:15,11:15,12:15,13:15,14:15,15:15,16:15,17:15,18:15,19:15,20:15,21:15,22:15,23:15},
-    'v':{0:15,1:15,2:15,3:15,4:15,5:15,6:15,7:15,8:15,9:15,10:15,11:15,12:15,13:15,14:15,15:15,16:15,17:15,18:15,19:15,20:15,21:15,22:15,23:15,24:15,25:15,26:15,27:15,28:15}
-    }
-    
-    def splitStacktoLines(locationall, rowcountarray):
-      locations = []
-      for i in range(len(rowcountarray) - 1):
-        locations.append(np.array(locationall[rowcountarray[i]:rowcountarray[i+1]]))
-      locations = np.array(locations)
-      return locations 
-    ii=0
-    V_index=0
-    xoff=0
-    maxyoff=0
-    tn=0
-    for tile in tilelist:#['h_l','h_m','h_h','v']:
-        tn+=1
-        tile=tile.replace(" ","")
-        linn=linedict[tile]
-        if tile!='v':
-            V_index+=linn
-            yoff=130
-        else:        
-            yoff=1
-            xoff=1
-        for i in range(linn):
-            if ii<30:
-                iko=ii
-            elif ii<84:
-                iko=ii-30            
-            elif ii<108:
-                iko=ii-84
-            else:
-                iko=ii-108
-            if ii>=78 and ii<108:
-                tn=3
-            fingerfile=str(port//10)
-            if port % 10 ==0:
-                 fingerfile=fingerfile+"l"+tile+"_"
-            else:
-                 fingerfile=fingerfile+"u"+tile+"_"
-            if port==20 and int(program.split(".")[0])<20171020:
-                linefile=fingerfile+'{0:02d}'.format(iko)+"2Dlines_biliniar_before_20171020.h5"
-            else:
-                linefile=fingerfile+'{0:02d}'.format(iko)+"2Dlines_biliniar.h5"
-            linf=h5.File(outpfad+linefile,'r')
-            linid=list(linf['LineNos'])
-            lines=linf['locationall2D']#
-            linlocid=linf['rowcountarray']
-            linrefpoi=linf['refPoi']
-            linweipoi=linf['weightPoi']
-            if port==50:
-                shapi=[1024, 1280]
-            else:
-                shapi=[768, 1024]
-            locations=splitStacktoLines(lines,linlocid)
-            locpoi=splitStacktoLines(linrefpoi,linlocid)
-            locwei=splitStacktoLines(linweipoi,linlocid)            
-            linf.close()
-            xoff+=1
-            if tile=='h_m' and iko<48 and iko%2==1:
-                shift=True
-                xoff=xoff-numstacks[tile][i-1]-1
-                maxyoff+=1
-            else:
-                shift=False
-                maxyoff=0
-            for j in range(numstacks[tile][i]): 
-                j2=linid.index(j)
-                if shift:
-                    yloff=maxyoff
-                else:
-                    yloff=0
-                lprof=np.array(File['profiles_'+str(ii)+"_"+str(j2)][0])
-                lalpha=np.array(File['alpha_'+str(ii)+"_"+str(j2)][0])
-                lheat=np.array(File['heat_'+str(ii)+"_"+str(j2)])
-                lXY=np.array(File['plot_profiles_'+str(ii)+"_"+str(j2)])
-                xf=np.array(np.asarray(locpoi[j2])%shapi[1],dtype=np.int16)
-                yf=np.array(np.asarray(locpoi[j2])//shapi[1],dtype=np.int16)
-                PX=np.asarray([np.dot(xf[o],locwei[j2][o]) for o in range(len(xf))])
-                PY=np.asarray([np.dot(yf[o],locwei[j2][o]) for o in range(len(yf))])
-                loco=locations[j2]
-                profilelength=int(np.max(lprof)/3e-3)+1
-                for k in range(profilelength):                
-                    pos=k*3e-3#3.143e-3                                        
-                    pid=IR.IR_tools.find_nearest(lprof,pos)
-                    if abs(pos-lprof[pid])<2e-3:
-                        alphapic[yoff+yloff,xoff]=lalpha[pid]
-                        heatpics[yoff+yloff,xoff,:]=lheat[:,pid]
-                        if upload_Mapping:
-                            Xpic[yoff+yloff,xoff]=lXY[pid,0]
-                            Ypic[yoff+yloff,xoff]=lXY[pid,1]
-                            FXpic[yoff+yloff,xoff]=loco[pid,0]
-                            FYpic[yoff+yloff,xoff]=loco[pid,1]
-                            profpic[yoff+yloff,xoff]=lprof[pid]
-        #                    profnpic[yoff+yloff,xoff]=j
-                            tilepic[yoff+yloff,xoff]=tn
-                            fingerpic[yoff+yloff,xoff]=ii*100+j
-                        if upload_scene_reference:
-                            PXpic[yoff+yloff,xoff]=int(round(PX[pid],0))
-                            PYpic[yoff+yloff,xoff]=int(round(PY[pid],0))
-                            Xpic3D[yoff+yloff,xoff]=Xloc[int(round(PY[pid],0)),int(round(PX[pid],0))]
-                            Ypic3D[yoff+yloff,xoff]=Yloc[int(round(PY[pid],0)),int(round(PX[pid],0))]
-                            Zpic3D[yoff+yloff,xoff]=Zloc[int(round(PY[pid],0)),int(round(PX[pid],0))]
-                            CID[yoff+yloff,xoff]=PFC[int(round(PY[pid],0)),int(round(PX[pid],0))]
-                    yloff+=1
-                xoff+=1
-                if yloff>maxyoff and not shift:
-                    maxyoff=yloff
-            ii+=1
-    del lalpha,lheat,lXY,loco,lprof,tn,File
-    #plt.figure();plt.imshow(alphapic[:,:,0])
-    #plt.figure();plt.imshow(tilepic[:,:,0])
-    #plt.figure();plt.imshow(fingerpic[:,:,0])
-    #plt.figure();plt.imshow(heatpics[:,:,50])
-    #testupload ####
-    
-    archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
-    prog=IR.get_program_from_PID(program)
-    if prog[0]:
-        t1=prog[1]['trigger']['1'][0]
-        time_ns=[int(t1-1e9)]
-        for t3 in time:
-            time_ns.append(int(t1+t3*1e9))
-        images=np.append(alphapic,heatpics,axis=2)
-        chanDesc=[{'name':'heatflux','physicalQuantity':{'type':'W/m^2'},'active':1}]
-        parobj={"label":"parms",
-                    "description":"heat flux for IR camera in AEF"+str(port),
-                    "values":[{"chanDescs":chanDesc,'header':header,'comment':comment}],
-                    "dimensions":[int(time_ns[0]),int(time_ns[-1])]}
-        
-        url=archivepath+project+"/"+group+"/AEF"+str(port)+"_heatflux"
-    #    url=IR.archivepath+project+"/"+group+"/"+stream
-        ver=IR.get_latest_version(group+"/"+stream+"_PARLOG",project=project,Test=True)
-        if ver==None:
-            reason="Versioning of Data"
-            IRup.archive_create_stream_version(url, reason, False, "n/a", "n/a" )
-        print(datetime.datetime.now(),"starting upload")
-#        IRup.upload_Parlog(url,parobj,versionnumber=1)  
-        print(datetime.datetime.now(),"parlog written")
-        IRup.upload_images(images,time=time_ns,group=group,stream=stream,project=project,version=1,swapping=False,Test=True)
-        print(datetime.datetime.now(),"data uploaded")
-        del images,alphapic,heatpics,time_ns
-        if upload_Mapping:
-            ### upload reference stream ###once!
-            time_ns2=[]
-            timet1=int(IR.TimeToNs([2017,9,26],[8,0,0,0]))
-            for t in range(7):
-                time_ns2.append(int(timet1+10*t))
-            images2=np.append(profpic,Xpic,axis=2)
-            images2=np.append(images2,Ypic,axis=2)
-            images2=np.append(images2,FXpic,axis=2)
-            images2=np.append(images2,FYpic,axis=2)
-            images2=np.append(images2,fingerpic,axis=2)    #    images2=np.append(images2,profnpic,axis=2)    
-            images2=np.append(images2,tilepic,axis=2)
-            header2={'profile':int(timet1),'MX':int(timet1+10),'MY':int(timet1+20),'FX':int(timet1+30),'FY':int(timet1+40),'finger_profile':int(timet1+50),'targetgroup':int(timet1+60)}
-            chanDesc=[{'name':'reference','physicalQuantity':{'type':'none'},'active':1}]
-            parobj={"label":"parms",
-                        "description":"reference data for the heat flux",
-                        "values":[{"chanDescs":chanDesc,'header':header2,'comment':comment}],
-                        "dimensions":[int(time_ns2[0]),int(time_ns2[-1])]
-                        }
-        #    url=IR.archivepath+project+"/"+group+"/"+stream2
-            url=archivepath+project+"/"+group+"/"+stream2
-            ver=IR.get_latest_version(group+"/"+stream2+"_PARLOG",project=project,Test=True)
-            if ver==None:
-                reason="Versioning of Data"
-                IRup.archive_create_stream_version(url, reason, False, "n/a", "n/a" )
-            IRup.upload_Parlog(url,parobj,versionnumber=1)  
-            IRup.upload_images(images2,time=time_ns2,group=group,stream=stream2,project=project,version=1,swapping=False,Test=True)
-        if upload_scene_reference:
-            ### upload scene model camera refrence stream (once per scene model)
-            time_ns_scene=[]
-            for t in range(6):
-                time_ns_scene.append(int(scene_reference_time+10*t))
-            images3=np.append(PXpic,PYpic,axis=2)
-            images3=np.append(images3,Xpic3D,axis=2)
-            images3=np.append(images3,Ypic3D,axis=2)
-            images3=np.append(images3,Zpic3D,axis=2)
-            images3=np.append(images3,CID,axis=2)
-            header3={'Pixel_X':int(scene_reference_time),'Pixel_Y':int(scene_reference_time+10),'CAD_X':int(scene_reference_time+20),'CAD_Y':int(scene_reference_time+30),'CAD_Z':int(scene_reference_time+40),'Component_ID':int(timet1+50)}
-            chanDesc=[{'name':'scene model reference','physicalQuantity':{'type':'none'},'active':1}]
-            parobj={"label":"parms",
-                        "description":"scene model reference data for the heat flux",
-                        "values":[{"chanDescs":chanDesc,'header':header3,'comment':comment}],
-                        "dimensions":[int(time_ns_scene[0]),int(time_ns_scene[-1])]
-                        }
-        #    url=IR.archivepath+project+"/"+group+"/"+stream2
-            url=archivepath+project+"/"+group+"/"+stream_scene
-            ver=IR.get_latest_version(group+"/"+stream_scene+"_PARLOG",project=project,Test=True)
-            if ver==None:
-                reason="Versioning of Data"
-                IRup.archive_create_stream_version(url, reason, False, "n/a", "n/a" )
-            IRup.upload_Parlog(url,parobj,versionnumber=1)  
-            IRup.upload_images(images3,time=time_ns_scene,group=group,stream=stream_scene,project=project,version=1,swapping=False,Test=True)
-            

upload_heatflux_folder.py

-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 21 09:32:29 2018
-V3.0.2
-@author: holn
-"""
-
-import numpy as np
-import scipy.io as sio
-#import matplotlib.pyplot as plt
-import h5py as h5
-import uploadingversionIRdata as IRup
-import downloadversionIRdata as IR
-import datetime
-import os
-import sys
-#125,190 vertical, horizontal *3 mm
-
-
-
-exception_ports=[]
-path="F:\\holn\\Documents\\Doktorarbeit\\RESULTS\\OP1.2b\\Fulldivertor_res\\DBM\\H\\"
-path="F:\\holn\\Documents\\Doktorarbeit\\Python-Programme\\THEODOR\\Divertor\\data\\"
-path="F:\\tmp\\upload_it\\"
-outpfad="F:\\holn\\Documents\\Doktorarbeit\\DATEN\\Mapping\\LinesforStacks\\"
-archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
-project="W7XAnalysis"
-group="QRT_IRCAM"
-  
-comment="Test upload"    
-header={'code_version':4,'alpha_corrected':1}
-stream2="Mapping_reference"
-upload_Mapping=False
-
-class Tee(object):
-    def __init__(self, *files):
-        self.files = files
-    def write(self, obj):
-        for f in self.files:
-            f.write(obj)
-            f.flush() # If you want the output to be visible immediately
-    def flush(self) :
-        for f in self.files:
-            f.flush()
-            
-original = sys.stdout
-start=datetime.datetime.now()
-fout=open("upload_log_heatflux_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
-#foutf=open("upload_log_failes_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
-sys.stdout = Tee(sys.stdout, fout)
-
-filelist=os.listdir(path)
-
-for file in filelist:
-    if "heatflux_V4.mat" in file:
-        program=file.split("_")[0]
-        port=int(file.split("_")[1].split("AEF")[1])
-        prog=IR.get_program_from_PID(program)
-        if prog[0]:
-            t1=prog[1]['trigger']['1'][0]
-            time_ns=[int(t1-1e9)]
-            try:
-#                import archivedb as AKF                
-                exist,tt,fra=IR.download_heatflux_by_times(port,t1,int(t1+100*1e6),version=1,request=False)            
-            except:
-                exist=False
-            if exist:
-                print(datetime.datetime.now(),"Data already uploaded for AEF"+str(port)+" , "+program)                
-            elif port not in exception_ports:
-                stream="AEF"+str(port)+"_heatflux"
-                print(datetime.datetime.now(),"loading "+program+"_AEF"+str(port)+"_heatflux_V4.mat")
-                File=sio.loadmat(path+program+"_AEF"+str(port)+"_heatflux_V4.mat")
-                time=np.asarray(File['time'][0])
-                tilelist=np.asarray(File['tiles'])
-                heatpics=np.zeros((125+190+10,108*12+4,len(time)),dtype=np.float32)
-                alphapic=np.zeros((125+190+10,108*12+4,1),dtype=np.float32)
-                alphapic.fill(np.nan)
-                heatpics.fill(np.nan)
-                Xpic=alphapic.copy()
-                Ypic=alphapic.copy()
-                FXpic=alphapic.copy()
-                FYpic=alphapic.copy()
-                profpic=alphapic.copy()
-                profnpic=alphapic.copy()
-                tilepic=alphapic.copy()
-                fingerpic=alphapic.copy()
-                
-                linedict={'h_l':30,'h_m':54,'h_h':24,'v':29}
-                numstacks={            
-                'h_l':{0:13,1:14,2:14,3:14,4:14,5:14,6:14,7:14,8:14,9:14,10:15,11:15,12:14,13:14,14:14,15:13,16:13,17:13,18:13,19:13,20:13,21:13,22:13,23:13,24:13,25:13,26:13,27:13,28:13,29:13},
-                'h_m':{0:14,1:15,2:14,3:15,4:14,5:15,6:14,7:15,8:14,9:15,10:14,11:15,12:14,13:15,14:14,15:15,16:14,17:15,18:14,19:15,20:14,21:15,22:14,23:15,24:14,25:15,26:14,27:15,28:14,29:15,30:14,31:15,32:14,33:15,34:14,35:15,36:14,37:15,38:14,39:15,40:14,41:15,42:14,43:15,44:14,45:15,46:14,47:15,48:15,49:14,50:14,51:14,52:14,53:15},
-                'h_h':{0:16,1:15,2:15,3:15,4:15,5:15,6:15,7:15,8:15,9:15,10:15,11:15,12:15,13:15,14:15,15:15,16:15,17:15,18:15,19:15,20:15,21:15,22:15,23:15},
-                'v':{0:15,1:15,2:15,3:15,4:15,5:15,6:15,7:15,8:15,9:15,10:15,11:15,12:15,13:15,14:15,15:15,16:15,17:15,18:15,19:15,20:15,21:15,22:15,23:15,24:15,25:15,26:15,27:15,28:15}
-                }
-                
-                def splitStacktoLines(locationall, rowcountarray):
-                  locations = []
-                  for i in range(len(rowcountarray) - 1):
-                    locations.append(np.array(locationall[rowcountarray[i]:rowcountarray[i+1]]))
-                  locations = np.array(locations)
-                  return locations 
-                ii=0
-                V_index=0
-                xoff=0
-                maxyoff=0
-                tn=0
-                for tile in tilelist:#['h_l','h_m','h_h','v']:
-                    tn+=1
-                    tile=tile.replace(" ","")
-                    linn=linedict[tile]
-                    if tile!='v':
-                        V_index+=linn
-                        yoff=130
-                    else:        
-                        yoff=1
-                        xoff=1
-                    for i in range(linn):
-                        if ii<30:
-                            iko=ii
-                        elif ii<84:
-                            iko=ii-30            
-                        elif ii<108:
-                            iko=ii-84
-                        else:
-                            iko=ii-108
-                        if ii>=78 and ii<108:
-                            tn=3
-                        fingerfile=str(port//10)
-                        if port % 10 ==0:
-                             fingerfile=fingerfile+"l"+tile+"_"
-                        else: