From 59cf0d0aa8b415ab7933fd83409ec93b299f72ed Mon Sep 17 00:00:00 2001
From: Holger Niemann <holger.niemann@ipp.mpg.de>
Date: Fri, 27 Oct 2017 12:49:37 +0200
Subject: [PATCH] add daily uploadscript
---
Upload_Day_IRdata.py | 426 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 426 insertions(+)
create mode 100644 Upload_Day_IRdata.py
diff --git a/Upload_Day_IRdata.py b/Upload_Day_IRdata.py
new file mode 100644
index 0000000..0e9fb68
--- /dev/null
+++ b/Upload_Day_IRdata.py
@@ -0,0 +1,426 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Sep 26 16:01:06 2017
+
+@author: holn
+"""
+
+import uploadingversionIRdata as IRup
+import os
+import h5py as h5
+import numpy as np
+import datetime
+import sys
+#from getfoldernames import Searchdatalocation as gfn
+
+#%% set global parameter
+
+is_local_test = True # local test on one IRcam PC (not central PC)
+active_PC = [0] # selection of the following camera PCs to loop over
+
+reupload=False
+reason=""
+retry=2
+
+singleupload=False#True#False
+singlelist=[]
+
+raw_parlog=True
+raw_data=True#False#True
+raw1=True#False#True#False#True
+raw2=True
+back_parlog=True#False#True
+background=True#False#True
+nuc_parlog=True#False
+nuc=True
+metastream=True
+
+dates=[[2017,10,11],[2017,10,12],[2017,10,17],[2017,10,18]]
+#date = [2017,9,26]
+
+ports = ['AEF10', 'AEF11',
+ 'AEF20', 'AEF21',
+ 'AEF30', 'AEF31',
+ 'AEF40', 'AEF41',
+ 'AEF50', 'AEF51']
+cameras = ['IRCAM', 'IRCAM',
+ 'IRCAM', 'IRCAM',
+ 'IRCAM', 'IRCAM',
+ 'IRCAM', 'IRCAM',
+ 'INFRATEC', 'IRCAM']
+Tup = [ False, True,
+ True, True,
+ True, False,
+ False, True,
+ False, True]
+pc_list = ['Pc-e4-qrt-7', 'Pc-e4-qrt-2',
+ 'Pc-e4-qrt-3', 'Pc-e4-qrt-4',
+ 'Pc-dia-ws-16', 'Pc-e4-qrt-6',
+ 'Pc-dia-ws-17', 'Pc-dia-ws-18',
+ 'Pc-e4-qrt-5', 'Pc-e4-qrt-1']
+
+toff = 0 #delay time of the programm for the raw stuff in ms
+
+#create upload log file:
+
+class Tee(object):
+ def __init__(self, *files):
+ self.files = files
+ def write(self, obj):
+ for f in self.files:
+ f.write(obj)
+ f.flush() # If you want the output to be visible immediately
+ def flush(self) :
+ for f in self.files:
+ f.flush()
+
+original = sys.stdout
+
+
+start=datetime.datetime.now()
+fout=open("upload_log_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
+foutf=open("upload_log_failes_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
+sys.stdout = Tee(sys.stdout, fout)
+#sys.stdout = fout
+
+#%% decide data path format
+for date in dates:
+ if date[0]==2017 and date[1]<=9 and date[2]<=25:
+ # old file and folder names
+ rawdatapath = "\\\\{0:s}\\r\\{1[0]:4d}-{1[1]:02d}-{1[2]:02d}"
+ calibdatapath = "\\\\{0:s}\\r\\Calibration\\{1[0]:4d}-{1[1]:02d}-{1[2]:02d}"
+ tempdatapath = "\\\\{0:s}\\t\\{1[0]:4d}-{1[1]:02d}-{1[2]:02d}"
+ """
+ have to be finished!!
+ """
+ else:
+ # recent file and folder naming convention
+ rawdatapath = "\\\\{0:s}\\r\\{1[0]:4d}.{1[1]:02d}.{1[2]:02d}"
+ calibdatapath = rawdatapath
+ tempdatapath = "\\\\{0:s}\\t\\{1[0]:4d}.{1[1]:02d}.{1[2]:02d}"
+
+ if is_local_test:
+ rawdatapath = "R:\\{1[0]:4d}.{1[1]:02d}.{1[2]:02d}"
+ calibdatapath = rawdatapath
+ tempdatapath = "T:\\{1[0]:4d}.{1[1]:02d}.{1[2]:02d}"
+
+ #%% loop over different IRcamera PC and upload data
+ if not raw1 and not raw2:
+ raw_parlog=False
+ raw_data=False
+
+ for i in active_PC:
+
+ port = int(ports[i].split('AEF')[1])
+ path = rawdatapath.format(pc_list[i],date)
+ print(datetime.datetime.now(),"Starting Upload for "+ports[i],date)
+ foutf.write(datetime.datetime.now(),"Starting Upload for "+ports[i],date,"\n")
+ # find all programs of this date on this PC
+ programtimes = []
+ if singleupload:
+ programtimes=singlelist
+ else:
+ for o in os.listdir(path):
+ if os.path.isdir(os.path.join(path,o)):
+ if len(o.split("_"))>1:
+ print("unsupported folderstructure at the moment!!")
+ else:
+ programtimes.append(o)
+
+ # # loop over all programs and upload data
+ for prog_time in programtimes:
+ # loop over one specific program for test and upload data
+ print("start upload of files for "+str(prog_time)+" for date: ",date)
+ foutf.write("start upload of files for "+str(prog_time)+" for date: ",date,"\n")
+ try:
+ path = os.path.join(rawdatapath.format(pc_list[i],date), prog_time)
+ # sort files into background, cold image, offset, gain and raw data
+ filelist = os.listdir(path)
+ blist = {} # background dic
+ clist = {} # cold file dic
+ olist = {} # offset file dic
+ glist = {} # gain file dic
+ rlist = {} # raw file dic
+ if len(filelist)>0:# check for empty folder!
+ for file in filelist:
+ dummy = file.split("_")
+ if dummy[1]=='background':
+ blist[dummy[7]] = file
+ elif dummy[1]=='nuc':
+ if dummy[2]=='cold':
+ clist[dummy[8]] = file
+ elif dummy[2]=='gain':
+ glist[dummy[8]] = file
+ elif dummy[2]=='offset':
+ olist[dummy[8]] = file
+ elif dummy[1]=='raw':
+ rlist[dummy[3]] = file
+ else:
+ print("unexpected file in the folder!")
+ foutf.write(datetime.datetime.now(),"unexpected file in the folder",prog_time,"\n")
+ print(datetime.datetime.now()," Start NUC upload")
+ # upload of NUC files, background files
+ if cameras[i]=="IRCAM":
+ newversion=reupload
+ if background or nuc:
+ for t_exp in range(4,10):
+ try:
+ print(datetime.datetime.now()," prepare upload of nuc data set for "+str(t_exp)+"us")
+ fb = h5.File(os.path.join(path, blist[str(t_exp)+"us"]),'r')
+ background_image = np.asarray(fb[list(fb.keys())[0]])
+ fb.close()
+ fg = h5.File(os.path.join(path, glist[str(t_exp)+"us"]),'r')
+ gain_image = np.asarray(fg[list(fg.keys())[0]])
+ fg.close()
+ fo = h5.File(os.path.join(path, olist[str(t_exp)+"us"]),'r')
+ offset_image = np.asarray(fo[list(fo.keys())[0]])
+ fo.close()
+ fc = h5.File(os.path.join(path, clist[str(t_exp)+"us"]),'r')
+ cold_image = np.asarray(fc[list(fc.keys())[0]])
+ fc.close()
+ bad = []
+ ftime = blist[str(t_exp)+"us"].split("_")[3].split(".")
+ ftime = list(map(int, ftime))
+ ftime[3] = ftime[3]*1000 #µs not ms!
+ time_stamp = int(IRup.TimeToNs(date,ftime))
+ #time_stamp = time_stamp #+ 6 * 6E10
+ if background:
+ try:
+ IRup.upload_Background_frame(cameras[i], port, [time_stamp], [t_exp],
+ background_image, newversion=newversion,reason=reason)
+ except Exception as E:
+ print("upload of background files for "+str(t_exp)+"us failed")
+ print(E)
+ foutf.write(datetime.datetime.now(),"upload of background files for "+str(t_exp)+"us failed",E,"\n")
+ else:
+ print("background upload deactivated")
+ foutf.write(datetime.datetime.now(),"background upload deactivated","\n")
+ if nuc:
+ try:
+ IRup.upload_NUC_ref(cameras[i], port, [time_stamp], [t_exp],
+ gain_image, offset_image, cold_image, bad,parlog=nuc_parlog,data=nuc,newversion=newversion,reason=reason)
+ except Exception as E:
+ print("upload of NUC files for "+str(t_exp)+"us failed")
+ print(E)
+ foutf.write(datetime.datetime.now(),"upload of NUC files for "+str(t_exp)+"us failed",E,"\n")
+ else:
+ print("nuc upload deactivated")
+ foutf.write(datetime.datetime.now(),"nuc upload deactivated","\n")
+ newversion=False#otherwise 6 new versions!
+ except Exception as E:
+ print(E)
+ foutf.write(datetime.datetime.now(),"error in reading",E,"\n")
+ else:
+ print("background and NUC upload deactivated")
+ foutf.write(datetime.datetime.now(),"background and NUC upload deactivated","\n")
+ # -->| tested --> works for IRCAM
+ else: #case for the Infratec files
+ newversion=reupload
+ fkeys=list(blist.keys())
+ print(datetime.datetime.now()," prepare upload of background data set for "+str(fkeys[0]))
+ filename=blist[fkeys[0]]
+ fb = h5.File(os.path.join(path, filename),'r')
+ background_image = np.asarray(fb[list(fb.keys())[0]])
+ ftime = filename.split("_")[3].split(".")
+ ftime = list(map(int, ftime))
+ ftime[3] = ftime[3]*1000 #µs not ms!
+ time_stamp = IRup.TimeToNs(date,ftime)
+
+ t_exp=int(filename.split("_")[7].split("us")[0])
+ cfilter=int(filename.split("_")[5])
+ if background:
+ try:
+ IRup.upload_Background_frame(cameras[i], port, [time_stamp], [t_exp,cfilter],
+ background_image, newversion=newversion,reason=reason)
+ except Exception as E:
+ print("upload of background file for "+str(t_exp)+"us failed")
+ print(E)
+ foutf.write(datetime.datetime.now(),"upload of background file for "+str(t_exp)+"us failed",E,"\n")
+ else:
+ print("background upload deactivated")
+ foutf.write(datetime.datetime.now(),"background upload deactivated","\n")
+
+ newversion=reupload
+ #%% ###upload now raw files###
+ fkeys=list(rlist.keys())
+ ftime1=fkeys[0].split(".")
+ ftime1 = list(map(int, ftime1))
+ ftime1[3]=ftime1[3]*1000#µs not ms!
+ ftime2=fkeys[1].split(".")
+ ftime2 = list(map(int, ftime2))
+ ftime2[3]=ftime2[3]*1000#µs not ms!
+ t1=int(IRup.TimeToNs(date,ftime1))
+ t2=int(IRup.TimeToNs(date,ftime2))
+ if t1>t2:
+ f1=1
+ f2=0
+ dummy=t2
+ t2=t1
+ t1=dummy
+ else:
+ f1=0
+ f2=1
+ # bla='09.34.44.100000'#'09.29.23.172000'
+ # bla=bla.split(".")
+ # bla = list(map(int, bla))
+ # t2=int(IRup.TimeToNs(date,bla))
+ hf1=h5.File(path+"\\"+rlist[fkeys[f1]],'r')
+ attri1=hf1['dynamic_attributes']
+ metadata1={
+ 'serialnumer': rlist[fkeys[f1]].split("_")[-1].split(".")[0],
+ 'width':int(np.asarray(attri1['ROI_w'])[0]),
+ 'heigth':int(np.asarray(attri1['ROI_h'])[0]),
+ 'pos_left':int(np.asarray(attri1['ROI_x'])[0]),
+ 'pos_right':int(np.asarray(attri1['ROI_y'])[0]),
+ 'bitdepth': int(np.asarray(attri1['bitdepth'])[0])
+ }
+ # tmeta1={'divertortemperature_K':np.asarray(attri1['divertor_temp_K'])[0],
+ # 'timestamp':np.asarray(attri1['timestamp'])[0]}
+ time1=np.asarray(hf1['timestamps'])
+ sensor1=np.asarray(attri1['sensor_temp_C'])
+ chip1=np.asarray(attri1['camera_temp_C'])
+ framerate1=np.asarray(attri1['frame_rate'])
+ expo1=np.asarray(attri1['integration_time_us'])
+ hf2=h5.File(path+"\\"+rlist[fkeys[f2]],'r')
+ attri2=hf2['dynamic_attributes']
+ metadata2={
+ 'serialnumer': rlist[fkeys[f1]].split("_")[-1].split(".")[0],
+ 'width':int(np.asarray(attri2['ROI_w'])[0]),
+ 'heigth':int(np.asarray(attri2['ROI_h'])[0]),
+ 'pos_left':int(np.asarray(attri2['ROI_x'])[0]),
+ 'pos_right':int(np.asarray(attri2['ROI_y'])[0]),
+ 'bitdepth': int(np.asarray(attri2['bitdepth'])[0])
+ }
+
+ # tmeta2={'divertortemperature_K':np.asarray(attri2['divertor_temp_K'])[0],
+ # 'timestamp':np.asarray(attri2['timestamp'])[0]}
+ time2=np.asarray(hf2['timestamps'])
+ sensor2=np.asarray(attri2['sensor_temp_C'])
+ chip2=np.asarray(attri2['camera_temp_C'])
+ framerate2=np.asarray(attri2['frame_rate'])
+ expo2=np.asarray(attri2['integration_time_us'])
+ if cameras[i]=="IRCAM":
+ metadata1['cameraname']= 'IRCam_Caleo768kL'
+ metadata2['cameraname']= 'IRCam_Caleo768kL'
+ metadata1['purpose']='NUC+background'
+ metadata2['purpose']='experiment'
+ #['ROI_h', 'ROI_w', 'ROI_x', 'ROI_y', 'Size',
+ #'bitdepth', 'camera_temp_C', 'divertor_temp_K', 'frame_number',
+ #'frame_rate', 'integration_time_us', 'sensor_temp_C', 'timestamp']
+ else:
+ metadata1['cameraname']= 'INFRATEC_IMAGEIR9300'
+ metadata1['filter']=int(np.asarray(attri1['filter'])[0])
+ #metadata1['multi_exposure']=np.asarray(attri1['multi_exposure'])[0]
+ metadata2['cameraname']= 'INFRATEC_IMAGEIR9300'
+ metadata2['filter']=int(np.asarray(attri2['filter'])[0])
+ metadata1['purpose']='background'
+ metadata2['purpose']='experiment'
+ #metadata2['multi_exposure']=np.asarray(attri2['multi_exposure'])[0]
+ hf1.close()
+ del attri1
+ hf2.close()
+ del attri2
+ print(datetime.datetime.now()," regenerate timevectors")
+ time1=IRup.regenerate_timevector(time1,0,t2,0)
+ time2=IRup.regenerate_timevector(time2,[1,6],t2,0)
+ print(datetime.datetime.now()," Upload Meta Stream(s)")
+ if metastream:
+ try:
+ IRup.upload_meta_stream(cameras[i],port,np.asarray(time1),sensor1,chip1,framerate1,expo1,newversion=newversion,reason=reason)
+ except Exception as E:
+ print(E)
+ foutf.write(datetime.datetime.now(),"metastream 1 upload failed",E,"\n")
+ del sensor1, chip1,framerate1,expo1
+ try:
+ IRup.upload_meta_stream(cameras[i],port,np.asarray(time2),sensor2,chip2,framerate2,expo2)
+ except Exception as E:
+ print(E)
+ foutf.write(datetime.datetime.now(),"metastream 2 upload failed",E,"\n")
+ del sensor2,chip2,framerate2,expo2
+ else:
+ print("metastream upload deactivated")
+ foutf.write(datetime.datetime.now(),"metastream upload deactivated","\n")
+ if raw_parlog or raw_data:
+ print(datetime.datetime.now()," Start Raw Data Upload for "+str(prog_time))
+ if raw1:
+ success=False
+ trys=0
+ while (trys<retry and not success):
+ try:
+ response=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time1,dtype=np.int64),path+"\\"+rlist[fkeys[f1]],metadata1,parlog=raw_parlog,rawdata=raw_data,newversion=newversion,reason=reason)#upload cold and backgrounds
+ if response[0]:
+ success=True
+ else:
+ print(datetime.datetime.now(),"raw 1 upload failed",response[1],response[2])
+ foutf.write(datetime.datetime.now(),"raw 1 upload failed",response[1],response[2],"\n")
+ except Exception as E:
+ trys+=1
+ print("Upload for rawfile 1 failed!, try:"+str(trys),E)
+ foutf.write(datetime.datetime.now(),"Upload for rawfile 1 failed!, try:"+str(trys),E,"\n")
+
+
+ if raw2:
+ success=False
+ trys=0
+ while (trys<retry and not success):
+ print(datetime.datetime.now()," Upload second file")
+ try:
+ response=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time2,dtype=np.int64),path+"\\"+rlist[fkeys[f2]],metadata2,parlog=raw_parlog,rawdata=raw_data)#upload raw images from plasma
+ if response[0]:
+ success=True
+ else:
+ print(datetime.datetime.now(),"raw 2 upload failed",response[1],response[2])
+ foutf.write(datetime.datetime.now(),"raw 2 upload failed",response[1],response[2],"\n")
+ except Exception as E:
+ trys+=1
+ print("Upload for rawfile 2 failed!, try"+str(trys),E)
+ foutf.write(datetime.datetime.now(),"Upload for rawfile 2 failed!, try:"+str(trys),E,"\n")
+
+ print(datetime.datetime.now()," Upload finished")
+ else:
+ print("raw upload deactivated")
+ foutf.write(datetime.datetime.now(),"raw upload deactivated","\n")
+ else:#folder is empty
+ print("no files in",prog_time)
+ reupload=False
+ except Exception as E:
+ print("Cannot Upload Data for folder "+str(prog_time))
+ print(E)
+ foutf.write(datetime.datetime.now(),"Cannot Upload Data for folder "+str(prog_time),"\n")
+ ###upload the temp files if allowed###
+ """
+ if Tup[i]:
+ path=tempdatapath.format(pc_list[i],date)+"\\"+prog_time
+ filelist=os.listdir(path)
+ #assumption that there are only temp files
+ fkeys=list(rlist.keys())
+ ftime1=fkeys[0].split(".")
+ for e in range(len(ftime1)):
+ ftime1[e]=int(ftime1[e])
+ ftime1[3]=ftime1[3]*1000#µs not ms!
+ ftime2=fkeys.split(".")
+ for e in range(len(ftime2)):
+ ftime2[e]=int(ftime2[e])
+ ftime2[3]=ftime2[3]*1000#µs not ms!
+ t1=IRup.TimeToNs(date,ftime1)
+ t2=IRup.TimeToNs(date,ftime2)
+ if t1>t2:
+ f1=1
+ f2=0
+ dummy=t2
+ t2=t1
+ t1=dummy
+ else:
+ f1=0
+ f2=1
+ IRup.upload_temperaure_data(cameras[i],port,images1,time1,tmeta1)
+ del images1, time1, tmeta1
+ IRup.upload_temperaure_data(cameras[i],port,images2,time2,tmeta2)
+ del images2, time2, tmeta2
+ #"""
+ ##[o for o in os.listdir(path) if os.path.isdir(os.path.join(path,o)) and (o[0]=='2')]
+ ##"{:02d}".format(zahl)
+
+fout.close()
+foutf.close()
+sys.stdout = original
\ No newline at end of file
--
GitLab