Skip to content

GitLab

Commit 7bdda71a authored by Holger Niemann's avatar Holger Niemann
Browse files

Merge branch 'master' into 'Holgers' 

Master

See merge request holn/IR_data_access!30
parents a24ffe93 08ec0353
Hide whitespace changes
Inline Side-by-side
downloadversionIRdata.py
View file @ 7bdda71a
......@@ -301,6 +301,8 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
if verbose>0:
print('get_NUC_by_times: calculate gain, offset and bad pixels')
gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
gain[gain == np.inf] = 0
offset[offset == np.inf] = 0
......@@ -309,6 +311,8 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
offset_error=0
return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
elif (port == 11) or (port == 21):
if verbose>0:
print('get_NUC_by_times: downloading NUC')
# coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
if gotit:
......@@ -329,6 +333,8 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
offset_error=0
return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
else:
if verbose>0:
print('get_NUC_by_times: downloading NUC')
return download_NUC_by_times(port,starttime,endtime,t_exp)
elif OP=="OP1.2b":
gain_error=0
......@@ -858,9 +864,9 @@ if fastDL:
images,error_images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
else:
images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
for i in range(len(images)):
images[i]=(images[i]*self.FOV).astype(np.float32)
if self.verbose>0:
print(datetime.datetime.now(),"NUC_raw_thread: correcting bad pixels")
images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
......@@ -1534,7 +1540,7 @@ def get_temp_from_raw_by_program_V1(portnr,program,time_window=0,emi=0.8,diverto
version=0,intervalSize=1E9,give_ERROR=False,verbose=0):
"""
"""
background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,1,version,verbose=verbose-1)
background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=1,version=version,verbose=verbose-1)
# prog=AKF_1.get_program_from_PID(program)
try:
t_program = AKF_2.get_program_from_to(program)
......@@ -1634,12 +1640,10 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=
FLIR=False
prog=get_program_from_PID(program)
if prog[0]:
t0=prog[1]['trigger']['0'][0]
t1=prog[1]['trigger']['1'][0]
t6=prog[1]['trigger']['6'][0]
success=True
OP=IR_tools.get_OP_by_time(time_ns=t1)
if type(portnr) is str:
try:
camera=portcamdict[OP][portnr]
......@@ -1651,9 +1655,10 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=
goon=True
except Exception as E:
raise Exception("get_temp_from_raw_by_program: unknown Port!"+E)
elif type(portnr) is int:
elif isinstance(portnr, (int, np.integer, float, np.float)):
FLIR=False
goon=True
portnr=int(portnr)
else:
goon=False
if not goon:
......@@ -1739,7 +1744,7 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=
print("get_temp_from_raw_by_program: Error! FLIR data not found")
return False,[0],[0],[]
else:
cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0,t1,t6,emi,T_version,version,back_emissivity,verbose=verbose-1)
cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,back_emissivity=back_emissivity,verbose=verbose-1)
if not cexist:
if verbose>0:
print("get_temp_from_raw_by_program: Unable to load the complete calibration data for",program," please confirm that this data was uploaded")
......@@ -1837,17 +1842,11 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.
elif type(portnr) is int:
FLIR=False
goon=True
portnr=int(portnr)
else:
goon=False
prog=get_program_from_PID(program)
if prog[0]:
t0=prog[1]['trigger']['0'][0]
t1=prog[1]['trigger']['1'][0]
t6=prog[1]['trigger']['6'][0]
else:
goon=False
if goon and not FLIR:
exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0,t1,t6,emi,T_version,version,verbose=verbose-1)
exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,verbose=verbose-1)
FOV=get_FOV_mask(portnr)
prog=get_program_from_PID(program)
if prog[0]:
......@@ -1939,11 +1938,12 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.
else:
return success,np.array(times),images,valid
def get_nuced_raw_by_program(portnr,program,time_window=0,version=0,threads=1,
def get_nuced_raw_by_program(portnr,program,time_window=0,version=0,
emi=1,T_version=2,threads=1,
give_ERROR=False,verbose=0):
"""
"""
background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi=1,T_version=2,version=version,verbose=verbose-1)
exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,verbose=verbose-1)
if verbose>0:
print(datetime.datetime.now(),"get_nuced_raw_by_program: raw download start")
if fastDL:
......@@ -1983,17 +1983,21 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8,
"""
prog=get_program_from_PID(program)
if prog[0]:
t0=prog[1]['trigger']['0'][0]
t1=prog[1]['trigger']['1'][0]
t6=prog[1]['trigger']['6'][0]
starttime=prog[1]['trigger']['1'][0]
stoptime=prog[1]['trigger']['6'][0]
if threads==1 or not fastDL:
return get_nuced_raw_by_program(portnr,program,time_window,emi,T_version,
version,threads,give_ERROR,verbose=verbose-1)
return get_nuced_raw_by_program(portnr,program,
time_window=time_window,emi=emi,
T_version=T_version,version=version,
threads=threads,give_ERROR=give_ERROR,
verbose=verbose-1)
else:
exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0,t1,t6,emi,T_version,version,verbose=verbose-1)
exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,verbose=verbose-1)
FOV=get_FOV_mask(portnr)
success=True
OP=IR_tools.get_OP_by_time(time_ns=t1)
OP=IR_tools.get_OP_by_time(time_ns=starttime)
if verbose>0:
print('get_nuced_raw_by_program_fullthreads: prepare loading images by threads')
if version==0:
version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
......@@ -2002,18 +2006,18 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8,
tstop=time_window[1]
if tstop<tstart:
raise Exception("get_nuced_raw_by_program_fullthreads: endtime before starttime")
stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
stdate=stdate.isoformat()
enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)
enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
enddate=enddate.isoformat()
else:
stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
stdate=stdate.isoformat()
if time_window==0:
enddate=datetime.datetime.utcfromtimestamp(t6/1e9)
enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)
enddate=enddate.isoformat()
else:
enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_window)
enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window)
enddate=enddate.isoformat()
times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
lnt=len(times)
......@@ -2031,7 +2035,10 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8,
if verbose>0:
print("get_nuced_raw_by_program_fullthreads: Start Thread ",i+1)
# p = multiprocessing.Process(target=NUC_raw_thread, args=(portnr,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,))
p = NUC_raw_thread(larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,verbose=verbose-1)
p = NUC_raw_thread(larchivepath,tim[intervalls[i]:intervalls[i+1]],
resultdict,i,version,background,LUT,refT,
gain,offset,gain_error,offset_error,give_ERROR,
FOV,badpixels,verbose=verbose-1)
jobs.append(p)
p.start()
for p in jobs:
......@@ -2175,27 +2182,33 @@ def get_cooling_water_temperature_by_time(time_ns):
result_out=[False,0,0]
return result_in[0] and result_out[0],inlet,outlet
def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivity=0.82,verbose=0):
def get_calib_data(port,program=None,t0=None,t1=None,t6=None,emissivity=0.8,
T_version=2,version=0,back_emissivity=0.82,verbose=0):
"""
get_calib_data
return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
"""
# prog=get_program_from_PID(program)
OP=IR_tools.get_OP_by_time(time_ns=t0)#program_str=program)
if t0 is None or t1 is None or t6 is None:
if program is None:
if t0 is not None:
program = AKF_2.get_program_id(t0)
elif t1 is not None:
program = AKF_2.get_program_id(t1)
else:
raise Exception('get_calib_data: ERROR! Missing time definition!')
print(' function requires either a program ID or the trigger times t0, t1 and t6')
# get start and end times from program ID
prog=get_program_from_PID(program)
if prog[0]:
t0=prog[1]['trigger']['0'][0]
t1=prog[1]['trigger']['1'][0]
t6=prog[1]['trigger']['6'][0]
else:
raise Exception('get_calib_data: ERROR! program {0} not found!'.format(program))
OP=IR_tools.get_OP_by_time(time_ns=t0)
exist=True
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
# if prog[0]:
# t0=prog[1]['trigger']['0'][0]
# t1=prog[1]['trigger']['1'][0]
# t6=prog[1]['trigger']['6'][0]
# except Exception as E:
# print("Warning: Program was not found: ",E)
# return 0,0,0,0,0,0,0,0,0,0
# else:
# raise Warning("cannot find the program!")
# return False,0,0,0,0,0,0,0,0,0,0
if verbose>0:
print('get_calib_data: loading data for camera {1} in {0}'.format(OP,port))
Camera=portcamdict[OP]['AEF'+str(port)]
......@@ -2219,18 +2232,44 @@ def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivi
if LUT_DL[0]:
LUT=LUT_DL[1]
del LUT_DL
if cfilter==0 and not Camera.split("_")[0]=="Infratec":
if cfilter==0 and not Camera.split("_")[0]=="Infratec":
if verbose>0:
print('get_calib_data: getting NUC arrays')
NUC_DL=get_NUC_by_times(port,t0,t1,t_exp,version,verbose=verbose-1)#download_NUC_by_times(port,t0,t1,t_exp,version)
if NUC_DL[0]==False:
raise Warning("get_calib_data: NUC was not found")
return False,0,LUT,0,0,0,0,t_exp,cfilter,0,0
else:
# extract NUC elements
gain = []
offset = []
badpixels = []
gain_error = []
offset_error = []
gain=np.array(NUC_DL[1][0])
offset=np.array(NUC_DL[1][1])
offset[offset<-1e300]=0
gain_error=np.array(NUC_DL[1][4])
offset_error=np.array(NUC_DL[1][5])
badpixels=np.array(NUC_DL[1][3],dtype=np.ubyte)
# check quality of NUC elements
badpixels[np.isnan(gain)] = 1
badpixels[np.isnan(offset)] = 1
gain[np.isnan(gain)]=0
offset[np.isnan(offset)]=0
badpixels[offset<-1e100] = 1
offset[offset<-1e100]=0
# catch cases of error frames being the value "0"
if NUC_DL[1][4] is 0:
gain_error = np.zeros(np.shape(gain), dtype=np.int8)
else:
gain_error=np.array(NUC_DL[1][4])
badpixels[np.isnan(gain_error)] = 1
gain_error[np.isnan(gain_error)]=np.nanmax(gain_error)+1
if NUC_DL[1][5] is 0:
offset_error = np.zeros(np.shape(offset), dtype=np.int8)
else:
offset_error=np.array(NUC_DL[1][5])
badpixels[np.isnan(offset_error)] = 1
offset_error[np.isnan(offset_error)]=np.nanmax(offset_error)+1
# catch special case for AEF11 and AEF21 in OP1.2a
if ((port == 11) or (port == 21)) and OP=="OP1.2a":
exist,time,frames=download_raw_images_by_times(port,t1,int(t1+0.02*1e9),version,verbose=verbose-1)
if exist:
......@@ -2254,7 +2293,7 @@ def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivi
raise Exception("get_calib_data: unknown Operation Phase or calibration method not implemented for this campaign")
if back_DL[0]:
background=back_DL[2]
if Temp_V==1:
if T_version==1:
if verbose>0:
print('get_calib_data: use temperature calibration version 1')
backtime=back_DL[1]
......@@ -2269,7 +2308,7 @@ def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivi
refT=np.average(TCT_dl[2])
else:
raise Exception("get_calib_data: Unable to find thermocouple data")
elif Temp_V==2:
elif T_version==2:
if verbose>0:
print('get_calib_data: use temperature calibration version 2')
frame=background.copy()
......@@ -2279,7 +2318,7 @@ def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivi
refT=waterout
else:
refT=28.5
elif Temp_V==3:
elif T_version==3:
if verbose>0:
print('get_calib_data: use temperature calibration version 3, TEST CASE!!!')
frame=background.copy()
......@@ -2326,6 +2365,8 @@ def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivi
init_bp_list = IR_tools.read_bad_pixels_from_file(port, time_ns=t1)
# find more bad pixel
badpixels = find_badpixels(port, gain, offset, init_bp_list=init_bp_list, niterations=10, tolerance=10,verbose=verbose-1)
if verbose>0:
print('get_calib_data: NUC arrays ready')
return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
# except Exception as E:
# print("Warning: Program was not found: ",E)
......@@ -2554,9 +2595,10 @@ def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode
goon=True
except Exception as E:
raise Exception("download_heatflux_by_times: unknown Port!"+E)
elif type(port) is int:
elif isinstance(port, (int, np.integer, float, np.float)):
FLIR=False
goon=True
port=int(port)
else:
goon=False
if not goon:
......@@ -2817,9 +2859,10 @@ def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstar
except Exception as E:
raise Warning("get_heatflux_profile: unknown Port!"+E)
return False,0,0,0
elif type(port) is int:
elif isinstance(port, (int, np.integer, float, np.float)):
FLIR=False
goon=True
port=port
else:
goon=False
if not goon or FLIR:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment