Commit 4bfd14ac authored by Holger Niemann's avatar Holger Niemann
Browse files

bug fixes and update of the upload script for more statistic and autodelete of uploaded data

parent 96c8b8df
......@@ -431,10 +431,10 @@ def calculate_gain_offset_image_pix(cold_image,hot_image=None,reference_cold=Non
Offset_rel = ( Off_h_rel + Off_c_rel ) /2
return Gain_rel,Offset_rel
def calculate_gain_offset_image(cold_image,hot_image=None,reference_cold=None,reference_hot=None,bose=1):
def calculate_gain_offset_image(cold_image,hot_image=None,reference_cold=None,reference_hot=None,verbose=0):
if hot_image is None:
hot_image=generate_new_hot_image(cold_image,reference_cold,reference_hot)
if bose>0:
if verbose>0:
print("calculate gain and offset")
# Sh_ref = hot_image[ ( np.int( np.shape(hot_image)[0] /2 ) ) ][np.int( (np.shape(hot_image)[1] /2 ) ) ]
......@@ -452,7 +452,6 @@ def calculate_gain_offset_image(cold_image,hot_image=None,reference_cold=None,re
Off_c_rel = Sc_ref - cold_image*Gain_rel
Offset_rel = ( Off_h_rel + Off_c_rel ) /2
return Gain_rel,Offset_rel
#%% functions from Yu Gao
""" functions by Yu Gao"""
......
......@@ -41,6 +41,7 @@ dates=[[2017,11,9]]
temperature=False
only_upload_lattest=False
use_today=False
delete_Files=False
logpath=""#"C:\\QIR\\Upload\\"
temppath=""#"C:\\QIR\\Upload\\"
......@@ -112,7 +113,12 @@ if delayupload:
original = sys.stdout
nrprograms=0
fully_uploaded=0
sum_meta1=0
sum_meta2=0
sum_raw1=0
sum_raw2=0
start=datetime.datetime.now()
fout=open(logpath+"upload_log_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
#foutf=open("upload_log_failes_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
......@@ -132,6 +138,7 @@ try:
Tup[ports.index(port)]=temperature
#%% decide data path format
for date in dates:
if date[0]==2017 and date[1]<=9 and date[2]<=25:
# old file and folder names
rawdatapath = "\\\\{0:s}\\r\\{1[0]:4d}-{1[1]:02d}-{1[2]:02d}"
......@@ -182,6 +189,7 @@ try:
# # loop over all programs and upload data
for prog_time in programtimes:
nrprograms+=1
# loop over one specific program for test and upload data
print("start upload of files for "+str(prog_time)+" for date: ",date)
# foutf.write("start upload of files for "+str(prog_time)+" for date: ",date,"\n")
......@@ -272,20 +280,20 @@ try:
# foutf.write(datetime.datetime.now(),"background and NUC upload deactivated","\n")
# -->| tested --> works for IRCAM
else: #case for the Infratec files
newversion=reupload
fkeys=list(blist.keys())
print(datetime.datetime.now()," prepare upload of background data set for "+str(fkeys[0]))
filename=blist[fkeys[0]]
fb = h5.File(os.path.join(path, filename),'r')
background_image = np.asarray(fb[list(fb.keys())[0]])
ftime = filename.split("_")[3].split(".")
ftime = list(map(int, ftime))
ftime[3] = ftime[3]*1000 #µs not ms!
time_stamp = IRup.TimeToNs(date,ftime)
t_exp=int(filename.split("_")[7].split("us")[0])
cfilter=int(filename.split("_")[5])
newversion=reupload
if background:
fkeys=list(blist.keys())
print(datetime.datetime.now()," prepare upload of background data set for "+str(fkeys[0]))
filename=blist[fkeys[0]]
fb = h5.File(os.path.join(path, filename),'r')
background_image = np.asarray(fb[list(fb.keys())[0]])
ftime = filename.split("_")[3].split(".")
ftime = list(map(int, ftime))
ftime[3] = ftime[3]*1000 #µs not ms!
time_stamp = IRup.TimeToNs(date,ftime)
t_exp=int(filename.split("_")[7].split("us")[0])
cfilter=int(filename.split("_")[5])
try:
IRup.upload_Background_frame(cameras[i], port, [time_stamp], [t_exp,cfilter],
background_image, newversion=newversion,reason=reason)
......@@ -468,7 +476,10 @@ try:
del attri1
hf2.close()
del attri2
metares1=0
metares2=0
rawres1=0
rawres2=0
print(datetime.datetime.now()," Upload Meta Stream(s)")
if metastream:
try:
......@@ -549,6 +560,26 @@ try:
rawres2=0
f_success.write(Program+" "+str(prog_time)+" "+str(metares1)+" "+str(metares2)+" "+str(rawres1)+" "+str(rawres2)+" \n")
f_success.flush()
sum_meta1+=metares1
sum_meta2+=metares2
sum_raw1+=rawres1
sum_raw2+=rawres2
if metares1==1 and metares2==metares1 and rawres1==rawres2 and rawres1==metares1:
fully_uploaded+=1
if delete_Files:
print("Upload of the 4 parts was successfull, deleting now the files in "+str(prog_time))
try:
os.remove(path+"\\"+rlist[fkeys[f1]])
except KeyboardInterrupt as ki: raise ki
except:
print('could not delete file "%s"' % path+"\\"+rlist[fkeys[f1]])
pass
try:
os.remove(path+"\\"+rlist[fkeys[f2]])
except KeyboardInterrupt as ki: raise ki
except:
print('could not delete file "%s"' % path+"\\"+rlist[fkeys[f2]])
pass
# foutf.write(datetime.datetime.now(),"raw upload deactivated","\n")
else:#folder is empty
print("no files in",prog_time)
......@@ -592,8 +623,14 @@ try:
##"{:02d}".format(zahl)
except Exception as E:
print("Upload failed! :",E)
print("script is done, the upload is to "+str(round(fully_uploaded/nrprograms*100,2))+"% completed; "+str(fully_uploaded)+" of "+str(nrprograms)+" folders are uploaded")
if fully_uploaded<nrprograms:
print("Calibration Meta Datastreams: "+str(sum_meta1)+" of "+str(nrprograms))
print("Plasma Meta Datastreams: "+str(sum_meta2)+" of "+str(nrprograms))
print("Calibration raw Datastreams: "+str(sum_raw1)+" of "+str(nrprograms))
print("plasma raw Datastreams: "+str(sum_raw2)+" of "+str(nrprograms))
fout.close()
f_success.close()
#foutf.close()
sys.stdout = original
char=input("script is done, press any button to close the program.")
\ No newline at end of file
char=input("script is done, press the enter button to close the program.")
\ No newline at end of file
......@@ -261,7 +261,7 @@ def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
else:
return False,0,0
def get_NUC_by_program(port,program,exposure,version=0):
def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
prog=AKF_1.get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
......@@ -269,13 +269,13 @@ def get_NUC_by_program(port,program,exposure,version=0):
if prog[0]:
starttime=prog[1]['trigger']['0'][0]
stoptime=prog[1]['trigger']['1'][0]
return get_NUC_by_times(port,starttime,stoptime,exposure,version=0)
return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose-1)
else:
# except:
print("cannot find the program")
return False,0,0
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0):
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
OP=IR_tools.get_OP_by_time(time_ns=endtime)
if OP=="OP1.2a":
if (port == 31) or (port == 21 and float(program[4:]) > 1110):
......@@ -295,7 +295,7 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0):
amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref)
gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
gain[gain == np.inf] = 0
offset[offset == np.inf] = 0
badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
......@@ -315,7 +315,7 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0):
raise Exception("NUC was not found")
else:
ci = NUC_DL[1][2]
gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref)
gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1)
gain[gain == np.inf] = 0
offset[offset == np.inf] = 0
badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
......@@ -342,7 +342,7 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0):
cold=np.asarray(cold/len(coldframes),dtype=np.uint16)
else:
return False,[0],[0]
gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref)
gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
else:
......@@ -449,15 +449,15 @@ def download_background_by_program(port,program,exposure,version=0):
'''
returned exist,time,frame
'''
# prog=AKF_1.get_program_from_PID(program)
try:
t_program = AKF_2.get_program_from_to(program)
prog =AKF_2.get_program_list(t_program[0], t_program[1])
# if prog[0]:
starttime=prog[0]['trigger']['0'][0]
stoptime=prog[0]['trigger']['1'][0]
prog=AKF_1.get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
if prog[0]:
starttime=prog[1]['trigger']['0'][0]
stoptime=prog[1]['trigger']['1'][0]
return download_background_by_times(port,starttime,stoptime,exposure,version)
except:
else:#except:
print("cannot find the program")
return False,0,0
......@@ -953,15 +953,16 @@ def get_sensor_temp_by_times(port,starttime,stoptime,version=0):
return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
def get_camera_framerate_by_program(port,program,version=0):
# prog=AKF_1.get_program_from_PID(program)
try:
t_program = AKF_2.get_program_from_to(program)
prog =AKF_2.get_program_list(t_program[0], t_program[1])
# if prog[0]:
starttime=prog[0]['trigger']['1'][0]
stoptime=prog[0]['trigger']['6'][0]
prog=AKF_1.get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
if prog[0]:
starttime=prog[1]['trigger']['1'][0]
stoptime=prog[1]['trigger']['6'][0]
return get_camera_framerate_by_times(port,starttime,stoptime,version)
except:
# except:
else:
print("cannot find the program")
return False,0,0
......@@ -1679,7 +1680,7 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
LUT=LUT_DL[1]
del LUT_DL
if cfilter==0 and not Camera.split("_")[0]=="Infratec":
NUC_DL=get_NUC_by_times(port,t0,t1,t_exp,version)#download_NUC_by_times(port,t0,t1,t_exp,version)
NUC_DL=get_NUC_by_times(port,t0,t1,t_exp,version,verbose=verbose-1)#download_NUC_by_times(port,t0,t1,t_exp,version)
if NUC_DL[0]==False:
raise Exception("NUC was not found")
else:
......@@ -1848,20 +1849,23 @@ def make_FOV_mask(port):
# inner_circle = mlt.patches.Circle((x0,y0), r1,color = 'r', fill = False)
# ax = fig.gca()
# ax.add_artist(inner_circle)
FOV=np.ones(np.shape(back))
for y in range(len(back)):
xs1=x0-np.sqrt(r1**2-(y-y0)**2)
xs2=x0+np.sqrt(r1**2-(y-y0)**2)
for xi in range(0,int(xs1)):
FOV[y][xi]=0
for xi in range(int(xs2),np.shape(back)[1]):
FOV[y][xi]=0
plt.figure()
plt.imshow(FOV*back,vmin=np.average(back)-500,vmax=np.average(back)+1500)
if config_path=="":
plt.imsave("AEF"+str(port)+"_FOV.png",FOV)
if da:
FOV=np.ones(np.shape(back))
for y in range(len(back)):
xs1=x0-np.sqrt(r1**2-(y-y0)**2)
xs2=x0+np.sqrt(r1**2-(y-y0)**2)
for xi in range(0,int(xs1)):
FOV[y][xi]=0
for xi in range(int(xs2),np.shape(back)[1]):
FOV[y][xi]=0
plt.figure()
plt.imshow(FOV*back,vmin=np.average(back)-500,vmax=np.average(back)+1500)
if config_path=="":
plt.imsave("AEF"+str(port)+"_FOV.png",FOV)
else:
plt.imsave(config_path+portcamdict['OP1.2a']['AEF'+str(port)]+"\\"+"AEF"+str(port)+"_FOV.png",FOV)
else:
plt.imsave(config_path+portcamdict['OP1.2a']['AEF'+str(port)]+"\\"+"AEF"+str(port)+"_FOV.png",FOV)
raise Exception("cannot find the background frame to create the FOV file")
def get_FOV_mask(port):
try:
......@@ -1936,10 +1940,19 @@ if __name__=='__main__':
program="20180712.027"
port=20
prog="20180724.030"
status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,1],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False)
port=51
program="20180809.010"
plt.figure()
for port in [10,11,20,21,30,31,40,41,51]:
framerate=get_camera_framerate_by_program(port,program)
if framerate[0]:
plt.plot(framerate[1],framerate[2],label="AEF"+str(port))
plt.legend()
plt.grid()
# status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,6],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
# status,time,images=download_raw_images_by_program_via_png(port,prog,time_s=1,threads=4,verbose=0)
test=get_NUC_by_program(port,prog,8,version=0)
# test=get_NUC_by_program(port,prog,8,version=0,verbose=5)
# print(len(images))
# import h5py as h5
# szene=h5.File("X:\\E4 Diagnostics\\QIR\Calibrations\\scene_models\\AEF"+str(port)+"\\AEF"+str(port)+"_scene_model.h5")
......@@ -1953,52 +1966,64 @@ if __name__=='__main__':
# cb=plt.colorbar()
# cb.set_label("temperature K", rotation=270,labelpad=26,fontsize=26)
# cb.ax.tick_params(labelsize=26)
plt.figure()# # plt.imshow(CAD,cmap='gray')
plt.imshow(images[-1],vmin=300,vmax=400,cmap=exJet)
cb=plt.colorbar()
cb.set_label("temperature K", rotation=270,labelpad=26,fontsize=26)
cb.ax.tick_params(labelsize=26)
# plt.figure()# # plt.imshow(CAD,cmap='gray')
# plt.imshow(images[-1],vmin=300,vmax=400,cmap=exJet)
# cb=plt.colorbar()
# cb.set_label("temperature K", rotation=270,labelpad=26,fontsize=26)
# proglist=["20180801.014","20180801.015","20180801.016","20180801.018","20180801.019"]
# portlist=[20,21,51]
# version=0
# time_s=30
# prog=AKF_1.get_program_from_PID(prog)
# if prog[0]:
# t0time=prog[1]['trigger']['0'][0]
# starttime=prog[1]['trigger']['1'][0]
# stoptime=prog[1]['trigger']['6'][0]
# success=True
# OP=get_OP_by_time(starttime)
# Cam=portcamdict[OP]['AEF'+str(port)]
# if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
# if version==0:
# version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
# larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
# elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
# if version==0:
# version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
# larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
# else:
# raise Exception("Port number does not fit the known cameras")
# if type(time_s)==list:
# tstart=time_s[0]
# tstop=time_s[1]
# if tstop<tstart:
# raise Exception("endtime before starttime")
# stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
# stdate=stdate.isoformat()
# enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
# enddate=enddate.isoformat()
# else:
# stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
# stdate=stdate.isoformat()
# t0date=datetime.datetime.utcfromtimestamp((t0time-100)/1e9)
# t0date=t0date.isoformat()
# if time_s==0:
# enddate=datetime.datetime.utcfromtimestamp(stoptime+100/1e9)
# enddate=enddate.isoformat()
# else:
# enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)
# enddate=enddate.isoformat()
# #"2017-11-15 08:00:00"
# times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
# timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),stdate.replace("T"," "))#
# print(len(timest0),len(times))
\ No newline at end of file
# time_s=30
# for program in proglist:
# for port in portlist:
# prog=AKF_1.get_program_from_PID(program)
# if prog[0]:
# t0time=prog[1]['trigger']['0'][0]
# starttime=prog[1]['trigger']['1'][0]
# try:
# stoptime=prog[1]['trigger']['6'][0]
# except:
# stoptime=prog[1]['upto']
# success=True
# OP=get_OP_by_time(starttime)
# Cam=portcamdict[OP]['AEF'+str(port)]
# if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
# if version==0:
# version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
# larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
# elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
# if version==0:
# version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
# larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
# else:
# raise Exception("Port number does not fit the known cameras")
# if type(time_s)==list:
# tstart=time_s[0]
# tstop=time_s[1]
# if tstop<tstart:
# raise Exception("endtime before starttime")
# stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
# stdate=stdate.isoformat()
# enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
# enddate=enddate.isoformat()
# else:
# stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
# stdate=stdate.isoformat()
# t0date=datetime.datetime.utcfromtimestamp((t0time-100)/1e9)
# t0date=t0date.isoformat()
# if time_s==0:
# enddate=datetime.datetime.utcfromtimestamp(stoptime+100/1e9)
# enddate=enddate.isoformat()
# else:
# enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)
# enddate=enddate.isoformat()
# #"2017-11-15 08:00:00"
# try:
# times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
# except:
# times=[]
# try:
# timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),stdate.replace("T"," "))#
# except:
# timest0=[]
# print(program,"AEF"+str(port),len(timest0),len(times))
\ No newline at end of file
......@@ -10,12 +10,12 @@ loading IR data and printing plots
import numpy as np
import downloadversionIRdata as downIR
import matplotlib.pyplot as plt
import datetime
if __name__=='__main__':
#%% loading data
status,time,images,valid=downIR.get_temp_from_raw_by_program(20,"20180726.015",time_s=2.5,emi=0.8,T_version=2,version=1,threads=4,give_ERROR=False)
print(datetime.datetime.now(),"start")
status,time,images,valid=downIR.get_temp_from_raw_by_program(10,"20180801.017",time_s=2.5,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,verbose=5)
print('fertig')
#%% plotting data
......@@ -33,5 +33,5 @@ if __name__=='__main__':
labelbottom='off') # labels along the bottom edge are off
c_ax = plt.colorbar()
c_ax.set_label('T [K]')
plt.savefig('20180726.015 - 2.2s - AEF20IR - in K.png', dpi=300, bbox_inches='tight')
plt.savefig('20180726.028 - 2.2s - AEF50IR - in K.png', dpi=300, bbox_inches='tight')
plt.show()
......@@ -4,14 +4,14 @@ local=True
delayupload=False
starttime="20:00:00"
singleupload=False
use_today=False
use_today=True
upload_only_lattest=False
dates=[[2017,11,9]]
singlelist=[]
### settings ###
logpath="C:\\QIR\\Upload\\"
temppath="C:\\QIR\\Upload\\"
delete_Files=False
delete_Files=True
rawdata=True
raw1=True
raw2=True
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment