Commit 58a1b396 authored by Holger Niemann's avatar Holger Niemann
Browse files

add function in IR_image_tools, bug fixes for OP1.2b upload

parent 1e12d13f
......@@ -88,6 +88,14 @@ def quad_abweich(data,fit):
print("Arrays must have same dimensions")
return R
def find_nearest(array,value):
#a=array
a = [x - value for x in array]
mini = np.min(np.abs(a))
try: idx= a.index(mini)
except: idx= a.index(-mini)
return idx#array[idx]
def check_coldframe(coldframe,references=None,threshold=0.5,plot_it=False):
'''
return true/false and the quality factor
......
......@@ -119,7 +119,7 @@ fout=open("upload_log_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minu
#foutf=open("upload_log_failes_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
sys.stdout = Tee(sys.stdout, fout)
#sys.stdout = fout
f_success=open("upload_log_successlist_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
#%% decide data path format
for date in dates:
if date[0]==2017 and date[1]<=9 and date[2]<=25:
......@@ -153,6 +153,8 @@ for date in dates:
port = int(ports[i].split('AEF')[1])
path = rawdatapath.format(pc_list[i],date)
print(datetime.datetime.now(),"Starting Upload for "+ports[i],date)
f_success.write(ports[i]+" \n")
f_success.write("program folder meta1 meta2 raw1 raw2 \n")
# foutf.write(datetime.datetime.now(),"Starting Upload for "+ports[i],date,"\n")
# find all programs of this date on this PC
programtimes = []
......@@ -245,7 +247,7 @@ for date in dates:
except Exception as E:
print("upload of NUC files for "+str(t_exp)+"us failed")
print(E)
foutf.write(datetime.datetime.now(),"upload of NUC files for "+str(t_exp)+"us failed",E,"\n")
# foutf.write(datetime.datetime.now(),"upload of NUC files for "+str(t_exp)+"us failed",E,"\n")
else:
print("nuc upload deactivated")
# foutf.write(datetime.datetime.now(),"nuc upload deactivated","\n")
......@@ -399,6 +401,24 @@ for date in dates:
frametype2.append(-1)
except:
frametype2=np.zeros((len(framenumber2)))-1
wrong1,correction_list1=IRup.check_timevector_framenumbers(time1,framenumber1)
wrong2,correction_list2=IRup.check_timevector_framenumbers(time2,framenumber2)
time1_0=time1.copy()
time2_0=time2.copy()
if wrong1:
sensor1=np.delete(sensor1,correction_list1)
chip1=np.delete(chip1,correction_list1)
framerate1=np.delete(framerate1,correction_list1)
frametype1=np.delete(frametype1,correction_list1)
expo1=np.delete(expo1,correction_list1)
time1=np.delete(time1,correction_list1)
if wrong2:
sensor2=np.delete(sensor2,correction_list2)
chip2=np.delete(chip2,correction_list2)
framerate2=np.delete(framerate2,correction_list2)
frametype2=np.delete(frametype2,correction_list2)
expo2=np.delete(expo2,correction_list2)
time2=np.delete(time2,correction_list1)
channel_names=["sensortemperature","cameratemperature","framerate","exposuretime","framenumber","frametype"]
channel_units=["oC","oC",'Hz','us','none','none']
channel_values1=[sensor1, chip1,framerate1,expo1,framenumber1,frametype1]
......@@ -415,8 +435,10 @@ for date in dates:
toff=0
print(datetime.datetime.now()," regenerate timevectors")
time1=IRup.regenerate_timevector(time1,0,t2,0)
time2=IRup.regenerate_timevector(time2,1,t2,toff)
time2,Program=IRup.regenerate_timevector(time2,1,t2,toff,True)
else:
correction_list1=[]
correction_list2=[]
channel_names=["sensortemperature","cameratemperature","framerate","exposuretime"]
channel_units=["oC","oC",'Hz','us']
channel_values1=[sensor1, chip1,framerate1,expo1]
......@@ -425,7 +447,9 @@ for date in dates:
comment_meta2=""
print(datetime.datetime.now()," regenerate timevectors")
time1=IRup.regenerate_timevector(time1,0,t2,0)
time2=IRup.regenerate_timevector(time2,[1,6],t2,0)
time2,Program=IRup.regenerate_timevector(time2,[1,6],t2,0,True)
time1_0=time1
time2_0=time2
hf1.close()
del attri1
hf2.close()
......@@ -434,19 +458,31 @@ for date in dates:
print(datetime.datetime.now()," Upload Meta Stream(s)")
if metastream:
try:
IRup.upload_meta_stream_V2(cameras[i],port,np.asarray(time1),channel_names,channel_units,channel_values1,newversion=newversion,reason=reason)
meta1up=IRup.upload_meta_stream_V2(cameras[i],port,np.asarray(time1),channel_names,channel_units,channel_values1,newversion=newversion,reason=reason, Return=True)
if meta1up:
metares1=1
else:
metares1=0
except Exception as E:
print(E)
metares1=0
# foutf.write(datetime.datetime.now(),"metastream 1 upload failed",E,"\n")
del sensor1, chip1,framerate1,expo1
try:
IRup.upload_meta_stream_V2(cameras[i],port,np.asarray(time2),channel_names,channel_units,channel_values2,newversion=newversion,reason=reason)
meta2up=IRup.upload_meta_stream_V2(cameras[i],port,np.asarray(time2),channel_names,channel_units,channel_values2,newversion=newversion,reason=reason, Return=True)
if meta2up:
metares2=1
else:
metares2=0
except Exception as E:
print(E)
metares2=0
# foutf.write(datetime.datetime.now(),"metastream 2 upload failed",E,"\n")
del sensor2,chip2,framerate2,expo2
else:
print("metastream upload deactivated")
metares1=0
metares2=0
# foutf.write(datetime.datetime.now(),"metastream upload deactivated","\n")
if raw_parlog or raw_data:
print(datetime.datetime.now()," Start Raw Data Upload for "+str(prog_time))
......@@ -455,38 +491,49 @@ for date in dates:
trys=0
while (trys<retry and not success):
try:
response=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time1,dtype=np.int64),path+"\\"+rlist[fkeys[f1]],metadata1,parlog=raw_parlog,rawdata=raw_data,newversion=newversion,reason=reason)#upload cold and backgrounds
response=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time1_0,dtype=np.int64),path+"\\"+rlist[fkeys[f1]],metadata1,parlog=raw_parlog,rawdata=raw_data,newversion=newversion,reason=reason,del_list=correction_list1)#upload cold and backgrounds
if response[0]:
success=True
rawres1=1
else:
print(datetime.datetime.now(),"raw 1 upload failed",response[1],response[2])
rawres1=0
# foutf.write(datetime.datetime.now(),"raw 1 upload failed",response[1],response[2],"\n")
except Exception as E:
trys+=1
print("Upload for rawfile 1 failed!, try:"+str(trys),E)
rawres1=0
# foutf.write(datetime.datetime.now(),"Upload for rawfile 1 failed!, try:"+str(trys),E,"\n")
else:
rawres1=0
if raw2:
success=False
trys=0
while (trys<retry and not success):
print(datetime.datetime.now()," Upload second file")
try:
response=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time2,dtype=np.int64),path+"\\"+rlist[fkeys[f2]],metadata2,parlog=raw_parlog,rawdata=raw_data)#upload raw images from plasma
response=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time2_0,dtype=np.int64),path+"\\"+rlist[fkeys[f2]],metadata2,parlog=raw_parlog,rawdata=raw_data,del_list=correction_list2)#upload raw images from plasma
if response[0]:
success=True
rawres2=1
else:
print(datetime.datetime.now(),"raw 2 upload failed",response[1],response[2])
rawres2=0
# foutf.write(datetime.datetime.now(),"raw 2 upload failed",response[1],response[2],"\n")
except Exception as E:
trys+=1
print("Upload for rawfile 2 failed!, try"+str(trys),E)
rawres2=0
# foutf.write(datetime.datetime.now(),"Upload for rawfile 2 failed!, try:"+str(trys),E,"\n")
else:
rawres2=0
print(datetime.datetime.now()," Upload finished")
else:
print("raw upload deactivated")
rawres1=0
rawres2=0
f_success.write(Program+" "+str(prog_time)+" "+str(metares1)+" "+str(metares2)+" "+str(rawres1)+" "+str(rawres2)+" \n")
# foutf.write(datetime.datetime.now(),"raw upload deactivated","\n")
else:#folder is empty
print("no files in",prog_time)
......@@ -530,5 +577,6 @@ for date in dates:
##"{:02d}".format(zahl)
fout.close()
f_success.close()
#foutf.close()
sys.stdout = original
\ No newline at end of file
......@@ -186,6 +186,29 @@ def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, ve
print("Warning: unable to find LUTs, check your request")
return False,0
def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
"""
read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
Read LUT V3 from local files for testing new calibration.
"""
LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
OP = get_OP_by_time(this_time)
cam_name = portcamdict[OP]['AEF{0}'.format(port)]
filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
# data = json.loads(join(LUT_dir, filename).decode('utf-8'))
try:
with open(LUT_dir+"\\"+filename) as data_file:
jfile=json.load(data_file)
LUT=np.array(jfile['LUT']).swapaxes(0,1)
if verbose>0:
print("succesfully loaded V3 LUT from local directory")
return True, LUT
except Exception as E:
print("ERROR in loading V3 LUTs",E)
return False, []
def download_NUC_by_program(port,program,exposure,version=0):
prog=AKF_1.get_program_from_PID(program)
# try:
......@@ -998,32 +1021,35 @@ def get_temp_from_raw_by_program_V1(portnr,program,time_s=0,emi=0.8,divertorpart
def get_temp_from_raw_by_program_V2(portnr,program,time_s=0,emi=0.8,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False):
return get_temp_from_raw_by_program(portnr,program,time_s=time_s,emi=emi,T_version=2,version=version,threads=threads,give_ERROR=give_ERROR,use_firstframe_as_background=use_firstframe_as_background)
def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,back_emissivity=0.8):
background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version,back_emissivity)
print(datetime.datetime.now(),"raw download start")
def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,back_emissivity=0.8,verbose=0):
background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version,back_emissivity,verbose=verbose-1)
if verbose>0:
print(datetime.datetime.now(),"raw download start")
if fastDL:
exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_s,version,threads)
exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_s,version,threads,verbose=verbose-1)
else:
exist,time,frames=download_raw_images_by_program(portnr,program,time_s,version)
exist,time,frames=download_raw_images_by_program(portnr,program,time_s,version,verbose=verbose-1)
if exist:
frames=[im.swapaxes(0,1) for im in frames]
else:
raise Exception("no data found")
print(datetime.datetime.now(),"raw download finished")
if verbose>0:
print(datetime.datetime.now(),"raw download finished")
FOV=get_FOV_mask(portnr)
if use_firstframe_as_background:
bim=apply_NUC([frames[0]],gain,offset)
background=get_average_background_recangle(portnr,bim[0])
if give_ERROR:
frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
else:
frames=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
frames=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
for i in range(len(frames)):
frames[i]=(frames[i]*FOV).astype(np.float32)
print(datetime.datetime.now(),"correcting bad pixels")
if verbose>0:
print(datetime.datetime.now(),"correcting bad pixels")
frames=IR_tools.correct_images(frames,badpixels)
print(datetime.datetime.now(),"checking temperaturerange")
if verbose>0:
print(datetime.datetime.now(),"checking temperaturerange")
valid=True
# for i in range(len(frames)):
try:
......@@ -1331,7 +1357,7 @@ def apply_NUC(images,gain,offset,gain_error=0,offset_error=0,give_error=False):
print(E)
return 0
def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivity=0.8, verbose=0):
def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivity=0.82, verbose=0):
prog=AKF_1.get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
......@@ -1389,10 +1415,27 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
refT=28.5
elif Temp_V==3:
if verbose>0:
print('use temperature calibration version 3')
print('use temperature calibration version 3, TEST CASE!!!')
print('use temperature calibration version 3, TEST CASE!!!')
frame=background.copy()
background=get_average_background_recangle(port,background)
refT=28.5
# read LUT V3 (compensated BG signal and photons flux of BG)
LUT_DL_wall = read_LUT_from_file(port, time, t_exp, 0.45, cfilter, verbose=verbose-1)
if LUT_DL_wall[0]:
if verbose>0:
print('got the wall LUT')
LUT_wall = LUT_DL_wall[1]
index=IR_tools.find_nearest(LUT_wall[1],refT)
background=background-LUT_wall[0][index]
refT=0
del LUT_DL_wall,LUT_wall
LUT_DL = read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
if LUT_DL[0]:
if verbose>0:
print('got the V3 LUT')
LUT = LUT_DL[1]
del LUT_DL
# back_off=estimate_offset(port,program)
# background=(background-back_off)/(back_emissivity)+back_off
else:
......@@ -1622,9 +1665,10 @@ if __name__=='__main__':
# prog="20171012.035"
prog="20171005.024"
prog="20171109.056"
prog="20171207.050"
program="20180712.027"
# status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,6],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False)
# status,time,images=download_raw_images_by_program_via_png(port,prog,time_s=6,threads=4,verbose=10)
status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,1],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False)
status,time,images=download_raw_images_by_program_via_png(port,prog,time_s=1,threads=4,verbose=0)
# print(len(images))
# import h5py as h5
# szene=h5.File("X:\\E4 Diagnostics\\QIR\Calibrations\\scene_models\\AEF"+str(port)+"\\AEF"+str(port)+"_scene_model.h5")
......@@ -1638,52 +1682,52 @@ if __name__=='__main__':
# cb=plt.colorbar()
# cb.set_label("temperature K", rotation=270,labelpad=26,fontsize=26)
# cb.ax.tick_params(labelsize=26)
# plt.figure()# # plt.imshow(CAD,cmap='gray')
# plt.imshow(images[-1],vmin=300,vmax=600,cmap=exJet)
# cb=plt.colorbar()
# cb.set_label("temperature K", rotation=270,labelpad=26,fontsize=26)
# cb.ax.tick_params(labelsize=26)
version=0
time_s=30
prog=AKF_1.get_program_from_PID(program)
if prog[0]:
t0time=prog[1]['trigger']['0'][0]
starttime=prog[1]['trigger']['1'][0]
stoptime=prog[1]['trigger']['6'][0]
success=True
OP=get_OP_by_time(starttime)
Cam=portcamdict[OP]['AEF'+str(port)]
if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
if version==0:
version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
if version==0:
version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
else:
raise Exception("Port number does not fit the known cameras")
if type(time_s)==list:
tstart=time_s[0]
tstop=time_s[1]
if tstop<tstart:
raise Exception("endtime before starttime")
stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
stdate=stdate.isoformat()
enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
enddate=enddate.isoformat()
else:
stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
stdate=stdate.isoformat()
t0date=datetime.datetime.utcfromtimestamp((t0time-100)/1e9)
t0date=t0date.isoformat()
if time_s==0:
enddate=datetime.datetime.utcfromtimestamp(stoptime+100/1e9)
enddate=enddate.isoformat()
else:
enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)
enddate=enddate.isoformat()
#"2017-11-15 08:00:00"
times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),stdate.replace("T"," "))#
print(len(times),len(timest0))
\ No newline at end of file
plt.figure()# # plt.imshow(CAD,cmap='gray')
plt.imshow(images[-1],vmin=300,vmax=600,cmap=exJet)
cb=plt.colorbar()
cb.set_label("temperature K", rotation=270,labelpad=26,fontsize=26)
cb.ax.tick_params(labelsize=26)
# version=0
# time_s=30
# prog=AKF_1.get_program_from_PID(program)
# if prog[0]:
# t0time=prog[1]['trigger']['0'][0]
# starttime=prog[1]['trigger']['1'][0]
# stoptime=prog[1]['trigger']['6'][0]
# success=True
# OP=get_OP_by_time(starttime)
# Cam=portcamdict[OP]['AEF'+str(port)]
# if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
# if version==0:
# version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
# larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
# elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
# if version==0:
# version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
# larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
# else:
# raise Exception("Port number does not fit the known cameras")
# if type(time_s)==list:
# tstart=time_s[0]
# tstop=time_s[1]
# if tstop<tstart:
# raise Exception("endtime before starttime")
# stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
# stdate=stdate.isoformat()
# enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
# enddate=enddate.isoformat()
# else:
# stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
# stdate=stdate.isoformat()
# t0date=datetime.datetime.utcfromtimestamp((t0time-100)/1e9)
# t0date=t0date.isoformat()
# if time_s==0:
# enddate=datetime.datetime.utcfromtimestamp(stoptime+100/1e9)
# enddate=enddate.isoformat()
# else:
# enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)
# enddate=enddate.isoformat()
# #"2017-11-15 08:00:00"
# times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
# timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),stdate.replace("T"," "))#
# print(len(times),len(timest0))
\ No newline at end of file
......@@ -325,7 +325,7 @@ def upload_meta_stream(camera,portnum,time,sensortemp,cameratemp,framerate,expos
}
upload_fulldataset(url,parobj,dataobj,version)
def upload_meta_stream_V2(camera,portnum,time,channel_names,channel_units,channel_values,comment="",newversion=False,reason=""):
def upload_meta_stream_V2(camera,portnum,time,channel_names,channel_units,channel_values,comment="",newversion=False,reason="",Return=False):
"""
old: sensortemp,cameratemp,framerate,exposuretime
new: channels with names, units and values
......@@ -362,7 +362,8 @@ def upload_meta_stream_V2(camera,portnum,time,channel_names,channel_units,channe
"values":channels.tolist(),
"dimensions":time.tolist()
}
upload_fulldataset(url,parobj,dataobj,version)
if Return: return upload_fulldataset(url,parobj,dataobj,version,Return=Return)
else: upload_fulldataset(url,parobj,dataobj,version)
def upload_raw_data(camera,portnum,time,images,metadata={},comment="",newversion=False,reason=""):
"""
......@@ -404,7 +405,7 @@ def upload_raw_data(camera,portnum,time,images,metadata={},comment="",newversion
upload_Parlog(url,parobj,version)
upload_images(camera,portnum,images,time,"raw",newversion,reason)
def regenerate_timevector(time,reftrig=1,tref=0,toff=0):
def regenerate_timevector(time,reftrig=1,tref=0,toff=0,give_program=False):
"""
regenerate_timevector(time,reftrig,tref,toff)
(np.array) time: timevector of length t, 1d, in ns
......@@ -423,13 +424,17 @@ def regenerate_timevector(time,reftrig=1,tref=0,toff=0):
tref=prog[1].get('trigger').get(str(reftrig[0]))[0]
tref2=prog[1].get('trigger').get(str(reftrig[1]))[0]
ntime=np.asarray(match_time_vectors_lowlevel(time,tref+toff*1e6,tref2))
return ntime
if give_program:
return ntime,prog[1]['id']
else:
return ntime
else:
print("Program for this reference timestamp not found, cannot generate new timevector")
return np.asarray(time)
if give_program:
return np.asarray(time),"20000000.000"
def upload_raw_data_from_H5(camera,portnum,time,h5file,metadata={},comment="",parlog=True,rawdata=True,newversion=False,reason=""):
def upload_raw_data_from_H5(camera,portnum,time,h5file,metadata={},comment="",parlog=True,rawdata=True,newversion=False,reason="",del_list=[]):
"""
upload_raw_data(camera,portnum,images,time,metadata={},newversion=False,reason="")
(str) camera: string of cameraname "IRCAM" or "INFRATEC", is used for the path
......@@ -468,19 +473,21 @@ def upload_raw_data_from_H5(camera,portnum,time,h5file,metadata={},comment="",pa
upload_Parlog(url,parobj,version)
if rawdata:
print("writing done, creating upload H5file")
tmp=writeH5_from_File(datastream_name,h5file,'images',time)
tmp=writeH5_from_File(datastream_name,h5file,'images',time,double_list=del_list)
print("uploading H5file")
if type(tmp) is list:
n=1
for ele in tmp:
success=True
print("uploading part "+str(n)+" of "+str(len(tmp)))
n+=1
bla= uploadH5_Versioning(group,datastream_name,ele,version,True)
if bla[0]:
print("upload of part "+str(n-1)+" of "+str(len(tmp))+" was successfull")
else:
success=False
print("upload of part "+str(n-1)+" of "+str(len(tmp))+" failed")
return bla
return [success]
else:
return uploadH5_Versioning(group,datastream_name,tmp,version,True)
......@@ -693,19 +700,14 @@ def upload_hot_cold_reference(port,path):
"""
"""
# if port==50:
# camera="INFRATEC"
# else:
camera="IRCAM"
group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
group="QRT_IRCAM_TEST"#+camera#+"_TEST" # test part will be removed in final version
name=["raw"]
datastream_name="AEF"+str(port)+"_raw"
print("not ready")
def upload_scene_model(port,time_ns,path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Calibrations\\scene_models",newversion=False,reason=""):
# if port==50:
# camera="INFRATEC"
# else:
camera="IRCAM"
group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
name=["scene_model"]
......@@ -766,7 +768,7 @@ def download_LUT_tables(camera,port,time,texp=0,emissivity=0,Filter=0):
def upload_Parlog(url,parobj,versionnumber=0):
def upload_Parlog(url,parobj,versionnumber=0,Return=False):
# print('>upload_Parlog: json of parlog')
# print(json.dumps(parobj).encode("utf-8"))
if versionnumber==0:
......@@ -775,27 +777,39 @@ def upload_Parlog(url,parobj,versionnumber=0):
parlog_request = urllib.request.Request( url + "_PARLOG/V"+str(versionnumber), data = json.dumps(parobj).encode("utf-8"),headers = { "Content-type" : "application/json" } )
try:
parlog_response = urllib.request.urlopen(parlog_request)
if Return:
return parlog_response.getcode()//100 is 2
except urllib.error.HTTPError as ex:
msg = ex.read()
raise RuntimeError(msg)
msg = ex.read()
if Return: False
else:
raise RuntimeError(msg)
def upload_data(url,dataobj,versionnumber=0):
def upload_data(url,dataobj,versionnumber=0,Return=False):
if versionnumber==0:
datastream_request = urllib.request.Request( url + "_DATASTREAM",data=json.dumps(dataobj).encode("utf-8"), headers={ "Content-type":"application/json" } )
else:
datastream_request = urllib.request.Request( url + "_DATASTREAM/V"+str(versionnumber),data=json.dumps(dataobj).encode("utf-8"), headers={ "Content-type":"application/json" } )
try:
datastream_response = urllib.request.urlopen(datastream_request)
if Return: return datastream_response.getcode()//100 is 2
except urllib.error.HTTPError as ex:
msg = ex.read()
raise RuntimeError(msg)
def upload_fulldataset(url,parlog,data,versionnumber,bparlog=True,bdata=True):
def upload_fulldataset(url,parlog,data,versionnumber,bparlog=True,bdata=True,Return=False):
if bparlog:
upload_Parlog(url,parlog,versionnumber)
if Return:
res=upload_Parlog(url,parlog,versionnumber,Return)
else:
upload_Parlog(url,parlog,versionnumber,Return)
if bdata:
upload_data(url,data,versionnumber)
if Return:
res1=upload_data(url,data,versionnumber,Return)
else:
upload_data(url,data,versionnumber)
if bdata and bparlog and Return:
return (res and res1)
### from here on subfunctions
......@@ -915,7 +929,7 @@ def writeH5(stream,data,dimof,idx=None):
# g.create_dataset(dataname, data=data.tolist(), dtype=dtype,compression="gzip")
# return tmpfile
def writeH5_from_File(stream,filename,key,dimof,idx=None):
def writeH5_from_File(stream,filename,key,dimof,idx=None,double_list=[]):
"""
path=Path, data=numpy.array, dimof=list of long
data.shape = (width,height,time) or (width,height) if len(dim)==1
......@@ -932,8 +946,21 @@ def writeH5_from_File(stream,filename,key,dimof,idx=None):
tmpfile += ".h5"
with h5reader.File(tmpfile, 'w') as f:
g = f.create_group('data') # requires [row,col,time]
g.create_dataset('timestamps', data=list(dimof), dtype='uint64',compression="gzip")
f5.copy(key,f['/data'],name=stream)
if len(double_list)==0:
g.create_dataset('timestamps', data=list(dimof), dtype='uint64',compression="gzip")
f5.copy(key,f['/data'],name=stream)
else:
print("removing double frame for the upload HDF5")
shapi=f5[key].shape
dummy=np.zeros((shapi[0],shapi[1],shapi[2]),dtype=f5[key].dtype)
f5[key].read_direct(dummy)
dummy=np.delete(dummy,double_list,2)
dimof=np.delete(dimof,double_list)#done already earlier
g.create_dataset('timestamps', data=list(dimof), dtype='uint64',compression="gzip")
dset=g.create_dataset(stream,shape=(shapi[0],shapi[1],shapi[2]-len(double_list)),