diff --git a/__pycache__/IR_config_constants.cpython-36.pyc b/__pycache__/IR_config_constants.cpython-36.pyc
deleted file mode 100644
index 225fdbe5060b98d544786f1ccecbf0c35244f8a4..0000000000000000000000000000000000000000
Binary files a/__pycache__/IR_config_constants.cpython-36.pyc and /dev/null differ
diff --git a/__pycache__/IR_image_tools.cpython-36.pyc b/__pycache__/IR_image_tools.cpython-36.pyc
deleted file mode 100644
index f9cc1ae14ab9de1a0420d2be88bd8c426a0bb0f1..0000000000000000000000000000000000000000
Binary files a/__pycache__/IR_image_tools.cpython-36.pyc and /dev/null differ
diff --git a/__pycache__/downloadversionIRdata.cpython-36.pyc b/__pycache__/downloadversionIRdata.cpython-36.pyc
deleted file mode 100644
index a1277411396bcf5c6940f6c8cf62db7696bba8c1..0000000000000000000000000000000000000000
Binary files a/__pycache__/downloadversionIRdata.cpython-36.pyc and /dev/null differ
diff --git a/downloadversionIRdata.py b/downloadversionIRdata.py
index a49b778303718ed565f2eebf297c505c28458ea1..b90c6681e8926cbc751e429462bc93e75fb1cf10 100644
--- a/downloadversionIRdata.py
+++ b/downloadversionIRdata.py
@@ -301,6 +301,8 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
+ if verbose>0:
+ print('get_NUC_by_times: calculate gain, offset and bad pixels')
gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
gain[gain == np.inf] = 0
offset[offset == np.inf] = 0
@@ -309,6 +311,8 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
offset_error=0
return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
elif (port == 11) or (port == 21):
+ if verbose>0:
+ print('get_NUC_by_times: downloading NUC')
# coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
if gotit:
@@ -329,6 +333,8 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
offset_error=0
return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
else:
+ if verbose>0:
+ print('get_NUC_by_times: downloading NUC')
return download_NUC_by_times(port,starttime,endtime,t_exp)
elif OP=="OP1.2b":
gain_error=0
@@ -858,9 +864,9 @@ if fastDL:
images,error_images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
else:
images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
-
for i in range(len(images)):
images[i]=(images[i]*self.FOV).astype(np.float32)
+
if self.verbose>0:
print(datetime.datetime.now(),"NUC_raw_thread: correcting bad pixels")
images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
@@ -1534,7 +1540,7 @@ def get_temp_from_raw_by_program_V1(portnr,program,time_window=0,emi=0.8,diverto
version=0,intervalSize=1E9,give_ERROR=False,verbose=0):
"""
"""
- background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,1,version,verbose=verbose-1)
+ background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=1,version=version,verbose=verbose-1)
# prog=AKF_1.get_program_from_PID(program)
try:
t_program = AKF_2.get_program_from_to(program)
@@ -1634,12 +1640,10 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=
FLIR=False
prog=get_program_from_PID(program)
if prog[0]:
- t0=prog[1]['trigger']['0'][0]
t1=prog[1]['trigger']['1'][0]
t6=prog[1]['trigger']['6'][0]
success=True
OP=IR_tools.get_OP_by_time(time_ns=t1)
-
if type(portnr) is str:
try:
camera=portcamdict[OP][portnr]
@@ -1651,9 +1655,10 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=
goon=True
except Exception as E:
raise Exception("get_temp_from_raw_by_program: unknown Port!"+E)
- elif type(portnr) is int:
+ elif isinstance(portnr, (int, np.integer, float, np.float)):
FLIR=False
goon=True
+ portnr=int(portnr)
else:
goon=False
if not goon:
@@ -1739,7 +1744,7 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=
print("get_temp_from_raw_by_program: Error! FLIR data not found")
return False,[0],[0],[]
else:
- cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0,t1,t6,emi,T_version,version,back_emissivity,verbose=verbose-1)
+ cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,back_emissivity=back_emissivity,verbose=verbose-1)
if not cexist:
if verbose>0:
print("get_temp_from_raw_by_program: Unable to load the complete calibration data for",program," please confirm that this data was uploaded")
@@ -1837,17 +1842,11 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.
elif type(portnr) is int:
FLIR=False
goon=True
+ portnr=int(portnr)
else:
goon=False
- prog=get_program_from_PID(program)
- if prog[0]:
- t0=prog[1]['trigger']['0'][0]
- t1=prog[1]['trigger']['1'][0]
- t6=prog[1]['trigger']['6'][0]
- else:
- goon=False
if goon and not FLIR:
- exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0,t1,t6,emi,T_version,version,verbose=verbose-1)
+ exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,verbose=verbose-1)
FOV=get_FOV_mask(portnr)
prog=get_program_from_PID(program)
if prog[0]:
@@ -1939,11 +1938,12 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.
else:
return success,np.array(times),images,valid
-def get_nuced_raw_by_program(portnr,program,time_window=0,version=0,threads=1,
+def get_nuced_raw_by_program(portnr,program,time_window=0,version=0,
+ emi=1,T_version=2,threads=1,
give_ERROR=False,verbose=0):
"""
"""
- background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi=1,T_version=2,version=version,verbose=verbose-1)
+ exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,verbose=verbose-1)
if verbose>0:
print(datetime.datetime.now(),"get_nuced_raw_by_program: raw download start")
if fastDL:
@@ -1983,17 +1983,21 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8,
"""
prog=get_program_from_PID(program)
if prog[0]:
- t0=prog[1]['trigger']['0'][0]
- t1=prog[1]['trigger']['1'][0]
- t6=prog[1]['trigger']['6'][0]
+ starttime=prog[1]['trigger']['1'][0]
+ stoptime=prog[1]['trigger']['6'][0]
if threads==1 or not fastDL:
- return get_nuced_raw_by_program(portnr,program,time_window,emi,T_version,
- version,threads,give_ERROR,verbose=verbose-1)
+ return get_nuced_raw_by_program(portnr,program,
+ time_window=time_window,emi=emi,
+ T_version=T_version,version=version,
+ threads=threads,give_ERROR=give_ERROR,
+ verbose=verbose-1)
else:
- exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0,t1,t6,emi,T_version,version,verbose=verbose-1)
+ exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,verbose=verbose-1)
FOV=get_FOV_mask(portnr)
success=True
- OP=IR_tools.get_OP_by_time(time_ns=t1)
+ OP=IR_tools.get_OP_by_time(time_ns=starttime)
+ if verbose>0:
+ print('get_nuced_raw_by_program_fullthreads: prepare loading images by threads')
if version==0:
version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
@@ -2002,18 +2006,18 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8,
tstop=time_window[1]
if tstop<tstart:
raise Exception("get_nuced_raw_by_program_fullthreads: endtime before starttime")
- stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
+ stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
stdate=stdate.isoformat()
- enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)
+ enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
enddate=enddate.isoformat()
else:
- stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
+ stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
stdate=stdate.isoformat()
if time_window==0:
- enddate=datetime.datetime.utcfromtimestamp(t6/1e9)
+ enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)
enddate=enddate.isoformat()
else:
- enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_window)
+ enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window)
enddate=enddate.isoformat()
times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
lnt=len(times)
@@ -2031,7 +2035,10 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8,
if verbose>0:
print("get_nuced_raw_by_program_fullthreads: Start Thread ",i+1)
# p = multiprocessing.Process(target=NUC_raw_thread, args=(portnr,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,))
- p = NUC_raw_thread(larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,verbose=verbose-1)
+ p = NUC_raw_thread(larchivepath,tim[intervalls[i]:intervalls[i+1]],
+ resultdict,i,version,background,LUT,refT,
+ gain,offset,gain_error,offset_error,give_ERROR,
+ FOV,badpixels,verbose=verbose-1)
jobs.append(p)
p.start()
for p in jobs:
@@ -2175,27 +2182,33 @@ def get_cooling_water_temperature_by_time(time_ns):
result_out=[False,0,0]
return result_in[0] and result_out[0],inlet,outlet
-def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivity=0.82,verbose=0):
+def get_calib_data(port,program=None,t0=None,t1=None,t6=None,emissivity=0.8,
+ T_version=2,version=0,back_emissivity=0.82,verbose=0):
"""
get_calib_data
return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
"""
-# prog=get_program_from_PID(program)
- OP=IR_tools.get_OP_by_time(time_ns=t0)#program_str=program)
+ if t0 is None or t1 is None or t6 is None:
+ if program is None:
+ if t0 is not None:
+ program = AKF_2.get_program_id(t0)
+ elif t1 is not None:
+ program = AKF_2.get_program_id(t1)
+ else:
+ raise Exception('get_calib_data: ERROR! Missing time definition!')
+ print(' function requires either a program ID or the trigger times t0, t1 and t6')
+ # get start and end times from program ID
+ prog=get_program_from_PID(program)
+ if prog[0]:
+ t0=prog[1]['trigger']['0'][0]
+ t1=prog[1]['trigger']['1'][0]
+ t6=prog[1]['trigger']['6'][0]
+ else:
+ raise Exception('get_calib_data: ERROR! program {0} not found!'.format(program))
+
+ OP=IR_tools.get_OP_by_time(time_ns=t0)
exist=True
-# try:
-# t_program = AKF_2.get_program_from_to(program)
-# prog =AKF_2.get_program_list(t_program[0], t_program[1])
-# if prog[0]:
-# t0=prog[1]['trigger']['0'][0]
-# t1=prog[1]['trigger']['1'][0]
-# t6=prog[1]['trigger']['6'][0]
-# except Exception as E:
-# print("Warning: Program was not found: ",E)
-# return 0,0,0,0,0,0,0,0,0,0
-# else:
-# raise Warning("cannot find the program!")
-# return False,0,0,0,0,0,0,0,0,0,0
+
if verbose>0:
print('get_calib_data: loading data for camera {1} in {0}'.format(OP,port))
Camera=portcamdict[OP]['AEF'+str(port)]
@@ -2219,18 +2232,44 @@ def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivi
if LUT_DL[0]:
LUT=LUT_DL[1]
del LUT_DL
- if cfilter==0 and not Camera.split("_")[0]=="Infratec":
+ if cfilter==0 and not Camera.split("_")[0]=="Infratec":
+ if verbose>0:
+ print('get_calib_data: getting NUC arrays')
NUC_DL=get_NUC_by_times(port,t0,t1,t_exp,version,verbose=verbose-1)#download_NUC_by_times(port,t0,t1,t_exp,version)
if NUC_DL[0]==False:
raise Warning("get_calib_data: NUC was not found")
return False,0,LUT,0,0,0,0,t_exp,cfilter,0,0
else:
+ # extract NUC elements
+ gain = []
+ offset = []
+ badpixels = []
+ gain_error = []
+ offset_error = []
gain=np.array(NUC_DL[1][0])
offset=np.array(NUC_DL[1][1])
- offset[offset<-1e300]=0
- gain_error=np.array(NUC_DL[1][4])
- offset_error=np.array(NUC_DL[1][5])
badpixels=np.array(NUC_DL[1][3],dtype=np.ubyte)
+ # check quality of NUC elements
+ badpixels[np.isnan(gain)] = 1
+ badpixels[np.isnan(offset)] = 1
+ gain[np.isnan(gain)]=0
+ offset[np.isnan(offset)]=0
+ badpixels[offset<-1e100] = 1
+ offset[offset<-1e100]=0
+ # catch cases of error frames being the value "0"
+ if NUC_DL[1][4] is 0:
+ gain_error = np.zeros(np.shape(gain), dtype=np.int8)
+ else:
+ gain_error=np.array(NUC_DL[1][4])
+ badpixels[np.isnan(gain_error)] = 1
+ gain_error[np.isnan(gain_error)]=np.nanmax(gain_error)+1
+ if NUC_DL[1][5] is 0:
+ offset_error = np.zeros(np.shape(offset), dtype=np.int8)
+ else:
+ offset_error=np.array(NUC_DL[1][5])
+ badpixels[np.isnan(offset_error)] = 1
+ offset_error[np.isnan(offset_error)]=np.nanmax(offset_error)+1
+ # catch special case for AEF11 and AEF21 in OP1.2a
if ((port == 11) or (port == 21)) and OP=="OP1.2a":
exist,time,frames=download_raw_images_by_times(port,t1,int(t1+0.02*1e9),version,verbose=verbose-1)
if exist:
@@ -2254,7 +2293,7 @@ def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivi
raise Exception("get_calib_data: unknown Operation Phase or calibration method not implemented for this campaign")
if back_DL[0]:
background=back_DL[2]
- if Temp_V==1:
+ if T_version==1:
if verbose>0:
print('get_calib_data: use temperature calibration version 1')
backtime=back_DL[1]
@@ -2269,7 +2308,7 @@ def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivi
refT=np.average(TCT_dl[2])
else:
raise Exception("get_calib_data: Unable to find thermocouple data")
- elif Temp_V==2:
+ elif T_version==2:
if verbose>0:
print('get_calib_data: use temperature calibration version 2')
frame=background.copy()
@@ -2279,7 +2318,7 @@ def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivi
refT=waterout
else:
refT=28.5
- elif Temp_V==3:
+ elif T_version==3:
if verbose>0:
print('get_calib_data: use temperature calibration version 3, TEST CASE!!!')
frame=background.copy()
@@ -2326,6 +2365,8 @@ def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivi
init_bp_list = IR_tools.read_bad_pixels_from_file(port, time_ns=t1)
# find more bad pixel
badpixels = find_badpixels(port, gain, offset, init_bp_list=init_bp_list, niterations=10, tolerance=10,verbose=verbose-1)
+ if verbose>0:
+ print('get_calib_data: NUC arrays ready')
return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
# except Exception as E:
# print("Warning: Program was not found: ",E)
@@ -2554,9 +2595,10 @@ def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode
goon=True
except Exception as E:
raise Exception("download_heatflux_by_times: unknown Port!"+E)
- elif type(port) is int:
+ elif isinstance(port, (int, np.integer, float, np.float)):
FLIR=False
goon=True
+ port=int(port)
else:
goon=False
if not goon:
@@ -2817,9 +2859,10 @@ def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstar
except Exception as E:
raise Warning("get_heatflux_profile: unknown Port!"+E)
return False,0,0,0
- elif type(port) is int:
+ elif isinstance(port, (int, np.integer, float, np.float)):
FLIR=False
goon=True
+ port=port
else:
goon=False
if not goon or FLIR: