From a9fa7d73d404319667cb3729a887494299c71a17 Mon Sep 17 00:00:00 2001
From: Holger Niemann <holger.niemann@ipp.mpg.de>
Date: Tue, 24 Jul 2018 17:54:27 +0200
Subject: [PATCH] implement OP1.2b conversion into download, bug fixing in
upload, a timevector error was inside
---
IR_image_tools.py | 4 +-
Upload_Day_IRdata.py | 49 ++---
downloadversionIRdata.py | 417 +++++++++++++++++++++++++++++++--------
3 files changed, 369 insertions(+), 101 deletions(-)
diff --git a/IR_image_tools.py b/IR_image_tools.py
index c77c872..0ced53c 100644
--- a/IR_image_tools.py
+++ b/IR_image_tools.py
@@ -439,8 +439,8 @@ def calculate_gain_offset_image(cold_image,hot_image=None,reference_cold=None,re
# Sh_ref = hot_image[ ( np.int( np.shape(hot_image)[0] /2 ) ) ][np.int( (np.shape(hot_image)[1] /2 ) ) ]
# Sc_ref = cold_image[ ( np.int( (np.shape(cold_image)[0]) /2 ) ) ][( np.int( (np.shape(cold_image)[1]) /2 ) ) ]
- print(hot_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])
- print(cold_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])
+# print(hot_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])
+# print(cold_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])
Sh_ref = np.mean( hot_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])
Sc_ref = np.mean(cold_image[( np.int( np.shape(cold_image)[0]/2) )-2: (np.int( np.shape(cold_image)[0]/2))+3,np.int((np.shape(cold_image)[1]/2))-2:np.int((np.shape(cold_image)[1]/2))+3 ])
Gain_rel = ( Sh_ref - Sc_ref ) / ( hot_image - cold_image)
diff --git a/Upload_Day_IRdata.py b/Upload_Day_IRdata.py
index b107e2c..472d79b 100644
--- a/Upload_Day_IRdata.py
+++ b/Upload_Day_IRdata.py
@@ -154,7 +154,9 @@ for date in dates:
path = rawdatapath.format(pc_list[i],date)
print(datetime.datetime.now(),"Starting Upload for "+ports[i],date)
f_success.write(ports[i]+" \n")
+ f_success.flush()
f_success.write("program folder meta1 meta2 raw1 raw2 \n")
+ f_success.flush()
# foutf.write(datetime.datetime.now(),"Starting Upload for "+ports[i],date,"\n")
# find all programs of this date on this PC
programtimes = []
@@ -374,31 +376,31 @@ for date in dates:
framenumber1=np.asarray(attri1['frame_number'])
framenumber2=np.asarray(attri2['frame_number'])
try:
- ftype1=np.asarray(attri1['frame_type'])
- frametype1=[]
- for ele in ftype1:
- if ele == "nuc":
- frametype1.append(0)
- elif ele == "background":
- frametype1.append(1)
- elif ele == "discharge":
- frametype1.append(2)
- else:
- frametype1.append(-1)
+ frametype1=np.asarray(attri1['frame_type'])
+# frametype1=[]
+# for ele in ftype1:
+# if ele == "nuc":
+# frametype1.append(0)
+# elif ele == "background":
+# frametype1.append(1)
+# elif ele == "discharge":
+# frametype1.append(2)
+# else:
+# frametype1.append(-1)
except:
frametype1=np.zeros((len(framenumber1)))-1
try:
- ftype2=np.asarray(attri2['frame_type'])
- frametype2=[]
- for ele in ftype2:
- if ele == "nuc":
- frametype2.append(0)
- elif ele == "background":
- frametype2.append(1)
- elif ele == "discharge":
- frametype2.append(2)
- else:
- frametype2.append(-1)
+ frametype2=np.asarray(attri2['frame_type'])
+# frametype2=[]
+# for ele in ftype2:
+# if ele == "nuc":
+# frametype2.append(0)
+# elif ele == "background":
+# frametype2.append(1)
+# elif ele == "discharge":
+# frametype2.append(2)
+# else:
+# frametype2.append(-1)
except:
frametype2=np.zeros((len(framenumber2)))-1
wrong1,correction_list1=IRup.check_timevector_framenumbers(time1,framenumber1)
@@ -436,6 +438,8 @@ for date in dates:
print(datetime.datetime.now()," regenerate timevectors")
time1=IRup.regenerate_timevector(time1,0,t2,0)
time2,Program=IRup.regenerate_timevector(time2,1,t2,toff,True)
+ time1_0=IRup.regenerate_timevector(time1_0,0,t2,0)
+ time2_0=IRup.regenerate_timevector(time2_0,0,t2,0)
else:
correction_list1=[]
correction_list2=[]
@@ -534,6 +538,7 @@ for date in dates:
rawres1=0
rawres2=0
f_success.write(Program+" "+str(prog_time)+" "+str(metares1)+" "+str(metares2)+" "+str(rawres1)+" "+str(rawres2)+" \n")
+ f_success.flush()
# foutf.write(datetime.datetime.now(),"raw upload deactivated","\n")
else:#folder is empty
print("no files in",prog_time)
diff --git a/downloadversionIRdata.py b/downloadversionIRdata.py
index 90be807..381aab6 100644
--- a/downloadversionIRdata.py
+++ b/downloadversionIRdata.py
@@ -224,6 +224,7 @@ def download_NUC_by_program(port,program,exposure,version=0):
return False,0,0
def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
+
larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_NUC_"
# NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if version==0:
@@ -258,6 +259,195 @@ def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
return False,0,0
else:
return False,0,0
+
+def get_NUC_by_program(port,program,exposure,version=0):
+ prog=AKF_1.get_program_from_PID(program)
+# try:
+# t_program = AKF_2.get_program_from_to(program)
+# prog =AKF_2.get_program_list(t_program[0], t_program[1])
+ if prog[0]:
+ starttime=prog[1]['trigger']['0'][0]
+ stoptime=prog[1]['trigger']['1'][0]
+ return get_NUC_by_times(port,starttime,stoptime,exposure,version=0)
+ else:
+# except:
+ print("cannot find the program")
+ return False,0,0
+
+def get_NUC_by_times(port,starttime,endtime,t_exp,version=0):
+ OP=IR_tools.get_OP_by_time(time_ns=endtime)
+ if OP=="OP1.2a":
+ if (port == 31) or (port == 21 and float(program[4:]) > 1110):
+ t1=endtime
+ t0=starttime
+ print("rebuilding coldframe")
+ #use any way the rebuilt coldframe.
+ sT = get_sensor_temp_by_program(port, program)[2][0]
+ # coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
+ gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
+ if gotit:
+ coldref=hotcold[1]
+ hotref=hotcold[0]
+ else:
+ raise Exception("unable to download reference frames")
+ filestring = 'AEF' + str(port) + '_et' + str(t_exp)
+ amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
+ bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
+ cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
+ gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref)
+ gain[gain == np.inf] = 0
+ offset[offset == np.inf] = 0
+ badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
+ gain_error=0
+ offset_error=0
+ return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
+ elif (port == 11) or (port == 21):
+ # coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
+ gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
+ if gotit:
+ coldref=hotcold[1]
+ hotref=hotcold[0]
+ else:
+ raise Exception("unable to download reference frames")
+ NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
+ if NUC_DL[0]==False:
+ raise Exception("NUC was not found")
+ else:
+ ci = NUC_DL[1][2]
+ gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref)
+ gain[gain == np.inf] = 0
+ offset[offset == np.inf] = 0
+ badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
+ gain_error=0
+ offset_error=0
+ return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
+ else:
+ return download_NUC_by_times(port,starttime,endtime,t_exp)
+ elif OP=="OP1.2b":
+ gain_error=0
+ offset_error=0
+ gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
+ if gotit:
+ coldref=hotcold[1]
+ hotref=hotcold[0]
+ else:
+ raise Exception("unable to download reference frames")
+ ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
+ exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
+ if exist:
+ cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
+ for ele in coldframes:
+ cold+=ele
+ cold=np.asarray(cold/len(coldframes),dtype=np.uint16)
+ else:
+ return False,[0],[0]
+ cold2=coldframes[0]
+ coldref2, hotref2 = IR_tools.load_ref_images('AEF' + str(port), t_exp)
+ gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref)
+ gain2, offset2 = IR_tools.calculate_gain_offset_image(cold2, None, coldref2, hotref2)
+ plt.imshow(offset,vmin=np.median(offset)-500,vmax=np.median(offset)+500)
+ plt.title("offset")
+ plt.figure()
+ plt.imshow(offset2,vmin=np.median(offset2)-500,vmax=np.median(offset2)+500)
+ plt.title("offset2")
+ badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
+ return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
+ else:
+ raise Exception("unknown Operation phase or NUC method not implemented for this OP")
+
+def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0):
+ """
+ download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
+ frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
+ """
+ gotit,time_t,texp_t=get_exposure_by_times(port,starttime,int(endtime-100))
+ if gotit:
+ expinds=np.where(texp_t==t_exp)[0]
+ if len(expinds)==0:
+ print("cannot find the exposure time in the given data")
+ return False,[0],[0]
+ else:
+ print("exposure time not found")
+ return False,[0],[0]
+ gotitf,timef,values_f=get_frametype_by_times(port,starttime,endtime)
+ if gotitf:
+ typiinds=np.where(values_f==frametype)[0]
+ if len(typiinds)>0:
+ ref_t=[np.min(timef[typiinds]),np.max(timef[typiinds])]
+ else:#okay the early data stuff or strange stuff
+ print("frame type was not identified assuming that the first part is the cold and the second one the background")
+ frametimes=time_t[expinds]
+ diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
+ turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
+ if frametype==0:
+ ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
+ elif frametype==1:
+# print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))
+ ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
+# print((ref_t[1]-ref_t[0])/1e9)
+ else:
+ raise Exception("requested Frametype unknown and not implemented!")
+ else:
+ print("frametype not found")
+ return False,[0],[0]
+ t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
+ t1date=t1date.isoformat()
+ t0date=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
+ t0date=t0date.isoformat()
+ if version==0:
+ version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_raw_DATASTREAM")
+ larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+ timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
+ t_offset=np.min(timest0)-time_t[0]
+ if t_offset>10e6:
+ print("time offset detected, try to correct this")
+ else:
+ t_offset=0
+# print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
+ return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset)
+
+def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
+ larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_raw_"
+# NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
+ if version==0:
+ version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_raw_DATASTREAM")
+ try:
+ res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)))
+ signal_list = json.loads(res.read().decode('utf-8'))
+ res.close()
+ goon=True
+ except urllib.error.URLError as e:
+ print(e)
+ goon=False
+ if goon:
+
+ COLDtime=0
+ HOTtime=0
+ try:
+ COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
+ except:
+ print("cold image for requested exposure time not found")
+ try:
+ HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
+ except:
+ print("hot image for requested exposure time not found")
+ images=[]
+ if HOTtime!=0:
+ HOT=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
+ if HOT[0]:
+ images.append(HOT[2])
+ if COLDtime!=0:
+ COLD=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
+ if COLD[0]:
+ images.append(COLD[2])
+
+ if HOT[0] and COLD[0] and len(images)==2:
+ return True,images,['hot','cold']
+ else:
+ print("hot and cold image for requested exposure time not found")
+ return False,0,0
+ else:
+ return False,0,0
def download_background_by_program(port,program,exposure,version=0):
'''
@@ -319,6 +509,32 @@ def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=
else:
return False,0,0
+def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=True):
+ "OP1.2b function"
+ exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version)
+ camera=portcamdict["OP1.2b"]["AEF"+str(port)]
+ background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
+ for frame in backgroundframes:
+ background+=frame
+ background=np.asarray(background/len(backgroundframes),dtype=np.uint16)
+ if plot_it:
+ plt.figure()
+ plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
+ plt.title("background image unnuced")
+ if not camera.split("_")[0]=="Infratec":
+ background=apply_NUC([background],gain,offset)
+ if plot_it:
+ plt.figure()
+ plt.imshow(gain,vmin=0,vmax=20)
+ plt.title("gain")
+ plt.figure()
+ plt.imshow(offset,vmin=np.median(offset)-500,vmax=np.median(offset)+500)
+ plt.title("offset")
+ plt.figure()
+ plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
+ plt.title("background image nuced")
+ return exist,btime[0],background
+
def download_raw_images_by_program(port,program,time_s=0,version=0):
# prog=AKF_1.get_program_from_PID(program)
try:
@@ -660,10 +876,13 @@ def get_exposure_by_program(port,program,version=0):
return False,0,0
def get_exposure_by_times(port,starttime,stoptime,version=0):
- if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+ OP=IR_tools.get_OP_by_time(time_ns=stoptime)
+ if OP=="OP1.2a" and port==50:
+# if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
- elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+# elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+ else:
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
if version==0:
@@ -750,6 +969,27 @@ def get_camera_framerate_by_times(port,starttime,stoptime,version=0):
return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
+def get_frametype_by_program(port,program,version=0):
+# prog=AKF_1.get_program_from_PID(program)
+ try:
+ t_program = AKF_2.get_program_from_to(program)
+ prog =AKF_2.get_program_list(t_program[0], t_program[1])
+# if prog[0]:
+ starttime=prog[0]['trigger']['1'][0]
+ stoptime=prog[0]['trigger']['6'][0]
+ return get_frametype_by_times(port,starttime,stoptime,version)
+ except:
+ print("cannot find the program")
+ return False,0,0
+
+def get_frametype_by_times(port,starttime,stoptime,version=0):
+ stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
+ larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
+ if version==0:
+ version=get_latest_version(stream+"DATASTREAM")
+ return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/5/frametype/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
+
+
def get_average_divertor_TC_for_camtemp(port,starttime,endtime,part="all",TC=0):
urlstart="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.115/"
if port==50 or port==51:
@@ -1035,7 +1275,7 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
raise Exception("no data found")
if verbose>0:
print(datetime.datetime.now(),"raw download finished")
- FOV=get_FOV_mask(portnr)
+ FOV=get_FOV_mask(portnr)
if use_firstframe_as_background:
bim=apply_NUC([frames[0]],gain,offset)
background=get_average_background_recangle(portnr,bim[0])
@@ -1058,6 +1298,16 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
print(E)
valid=False
# i=len(frames)
+ date=int(program.split(".")[0])
+ if date>20180712 and date<20180720:
+ prog=AKF_1.get_program_from_PID(program)
+ if prog[0]:
+# t0time=prog[1]['trigger']['0'][0]
+ starttime=prog[1]['trigger']['1'][0]
+ toff=time[0]-starttime
+ if toff>10e6:
+ print("time offset detected, offset is "+str(toff)+" ns, correcting")
+ time[:]=time[:]-toff
if give_ERROR:
return exist,time,frames,valid,error_images
else:
@@ -1356,9 +1606,40 @@ def apply_NUC(images,gain,offset,gain_error=0,offset_error=0,give_error=False):
print("error")
print(E)
return 0
+
+def get_cooling_water_temperature_by_time(time_ns):
+ starttime=int(time_ns-3e9)
+ endtime=int(time_ns+3e9)
+ outlet_url="95/Ist%20Av06_ABK10_CT006"
+ inlet_url="94/Ist%20Av06_ABK10_CT003"
+ url_base="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.85/DataModuleDesc.188_DATASTREAM/"
+ try:
+ result_in=AKF_1.read_restdb(url_base+inlet_url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
+ if result_in[0]:
+ inlet=np.mean(np.array(result_in[2]))
+ else:
+ print("inlet water temperature not found",result_in)
+ inlet=0
+ except:
+ print("inlet water temperature not found")
+ inlet=0
+ result_in=[False,0,0]
+ try:
+ result_out=AKF_1.read_restdb(url_base+outlet_url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
+ if result_out[0]:
+ outlet=np.mean(np.array(result_out[2]))
+ else:
+ print("outlet water temperature not found",result_out)
+ outlet=0
+ except:
+ print("outlet water temperature not found")
+ inlet=0
+ result_out=[False,0,0]
+ return result_in[0] and result_out[0],inlet,outlet
def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivity=0.82, verbose=0):
prog=AKF_1.get_program_from_PID(program)
+ OP=IR_tools.get_OP_by_time(program_str=program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
@@ -1369,7 +1650,7 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
# except Exception as E:
# print("Warning: Program was not found: ",E)
# return 0,0,0,0,0,0,0,0,0,0
- OP=get_OP_by_program(program)
+# OP=get_OP_by_program(program)
Camera=portcamdict[OP]['AEF'+str(port)]
if Camera.split("_")[0]=="Infratec":
cf=get_INFRATEC_filter_by_program(program,version)
@@ -1388,8 +1669,38 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
LUT_DL=download_LUT(port,time,t_exp,emissivity,cfilter,version, verbose=verbose-1)
if LUT_DL[0]:
LUT=LUT_DL[1]
- del LUT_DL
- back_DL=download_background_by_times(port,t0,t1,t_exp,cfilter,version)
+ del LUT_DL
+ if cfilter==0 and not Camera.split("_")[0]=="Infratec":
+ NUC_DL=get_NUC_by_times(port,t0,t1,t_exp,version)#download_NUC_by_times(port,t0,t1,t_exp,version)
+ if NUC_DL[0]==False:
+ raise Exception("NUC was not found")
+ else:
+ gain=np.array(NUC_DL[1][0])
+ offset=np.array(NUC_DL[1][1])
+ offset[offset<-1e300]=0
+ gain_error=np.array(NUC_DL[1][4])
+ offset_error=np.array(NUC_DL[1][5])
+ badpixels=np.array(NUC_DL[1][3],dtype=np.ubyte)
+ if ((port == 11) or (port == 21)) and OP=="OP1.2a":
+ exist,time,frames=download_raw_images_by_times(port,t1,int(t1+0.02*1e9),version)
+ if exist:
+ # frames=[im.swapaxes(0,1) for im in frames]#somehow the archive gives it now already swapped back (09.07.2018)
+ bim=apply_NUC([frames[0]],gain,offset)
+ background=get_average_background_recangle(port,bim[0])
+ else:
+ raise Exception("cannot find the first frame of the discharge, reconstruction failed")
+ else:
+ gain=0
+ offset=0
+ badpixels=0
+ gain_error=0
+ offset_error=0
+ if OP=="OP1.2a":
+ back_DL=download_background_by_times(port,t0,t1,t_exp,cfilter,version)
+ elif OP=="OP1.2b":
+ back_DL=get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version)
+ else:
+ raise Exception("unknown Operation Phase or calibration method not implemented for this campaign")
if back_DL[0]:
background=back_DL[2]
if Temp_V==1:
@@ -1412,7 +1723,11 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
print('use temperature calibration version 2')
frame=background.copy()
background=get_average_background_recangle(port,background)
- refT=28.5
+ gotit,waterin,waterout=get_cooling_water_temperature_by_time(t1)
+ if gotit:
+ refT=waterout
+ else:
+ refT=28.5
elif Temp_V==3:
if verbose>0:
print('use temperature calibration version 3, TEST CASE!!!')
@@ -1440,79 +1755,23 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
# background=(background-back_off)/(back_emissivity)+back_off
else:
raise Exception("Unknown temperature calibration method")
- if cfilter==0 and not Camera.split("_")[0]=="Infratec":
- if (port == 31) or (port == 21 and float(program[4:]) > 1110):
- print("rebuilding coldframe")
- #use any way the rebuilt coldframe.
- sT = get_sensor_temp_by_program(port, program)[2][0]
- coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
- filestring = 'AEF' + str(port) + '_et' + str(t_exp)
- amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
- bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
- cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
- gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref)
- gain[gain == np.inf] = 0
- offset[offset == np.inf] = 0
- badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
- gain_error=0
- offset_error=0
- exist,time,frames=download_raw_images_by_times(port,t1,int(t1+0.02*1e9),version)
- if exist:
-# frames=[im.swapaxes(0,1) for im in frames]#somehow the archive gives it now already swapped back (09.07.2018)
- bim=apply_NUC([frames[0]],gain,offset)
- background=get_average_background_recangle(port,bim[0])
- else:
- raise Exception("cannot find the first frame of the discharge, reconstruction failed")
- elif (port == 11) or (port == 21):
- coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
- NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
- if NUC_DL[0]==False:
- raise Exception("NUC was not found")
- else:
- ci = NUC_DL[1][2]
- gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref)
- gain[gain == np.inf] = 0
- offset[offset == np.inf] = 0
- badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
- gain_error=0
- offset_error=0
- exist,time,frames=download_raw_images_by_times(port,t1,int(t1+0.02*1e9),version)
- if exist:
-# frames=[im.swapaxes(0,1) for im in frames]
- bim=apply_NUC([frames[0]],gain,offset)
- background=get_average_background_recangle(port,bim[0])
- else:
- raise Exception("background wrong due to wrong NUC, firstframe not found, reconstruction failed")
- else:
- NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
- if NUC_DL[0]==False:
- raise Exception("NUC was not found")
- else:
- gain=np.array(NUC_DL[1][0])
- offset=np.array(NUC_DL[1][1])
- offset[offset<-1e300]=0
- gain_error=np.array(NUC_DL[1][4])
- offset_error=np.array(NUC_DL[1][5])
- badpixels=np.array(NUC_DL[1][3],dtype=np.ubyte)
- if np.max(badpixels)==0:
- if verbose>0:
- print(datetime.datetime.now(),"Scanning for bad pixel")
- # initial list from config files
- init_bp_list = IR_tools.read_bad_pixels_from_file(port, program=program)
- # find more bad pixel
- badpixels = find_badpixels(port, gain, offset, init_bp_list=init_bp_list, niterations=10, tolerance=10, verbose=verbose-1)
- else:
- gain=0
- offset=0
- badpixels=find_badpixels(port,frame-background,offset,niterations=10,tolerance=10,plot_it=False)#find_badpixels(port,gain,offset)
- gain_error=0
- offset_error=0
+
else:
raise Exception("no background image found")
else:
raise Exception("no LUT found")
else:
raise Exception("no exposure time found")
+ if np.max(badpixels)==0:
+ if Camera.split("_")[0]=="Infratec":
+ badpixels=find_badpixels(port,frame-background,offset,niterations=10,tolerance=10,plot_it=False)#find_badpixels(port,gain,offset)
+ else:
+ if verbose>0:
+ print(datetime.datetime.now(),"Scanning for bad pixel")
+ # initial list from config files
+ init_bp_list = IR_tools.read_bad_pixels_from_file(port, program=program)
+ # find more bad pixel
+ badpixels = find_badpixels(port, gain, offset, init_bp_list=init_bp_list, niterations=10, tolerance=10, verbose=verbose-1)
return background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
# except Exception as E:
# print("Warning: Program was not found: ",E)
@@ -1667,8 +1926,12 @@ if __name__=='__main__':
prog="20171109.056"
prog="20171207.050"
program="20180712.027"
+ port=11
+ prog="20180718.030"
status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,1],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False)
- status,time,images=download_raw_images_by_program_via_png(port,prog,time_s=1,threads=4,verbose=0)
+# status,time,images=download_raw_images_by_program_via_png(port,prog,time_s=1,threads=4,verbose=0)
+
+# test=get_NUC_by_program(port,prog,8,version=0)
# print(len(images))
# import h5py as h5
# szene=h5.File("X:\\E4 Diagnostics\\QIR\Calibrations\\scene_models\\AEF"+str(port)+"\\AEF"+str(port)+"_scene_model.h5")
@@ -1689,7 +1952,7 @@ if __name__=='__main__':
cb.ax.tick_params(labelsize=26)
# version=0
# time_s=30
-# prog=AKF_1.get_program_from_PID(program)
+# prog=AKF_1.get_program_from_PID(prog)
# if prog[0]:
# t0time=prog[1]['trigger']['0'][0]
# starttime=prog[1]['trigger']['1'][0]
@@ -1730,4 +1993,4 @@ if __name__=='__main__':
# #"2017-11-15 08:00:00"
# times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
# timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),stdate.replace("T"," "))#
-# print(len(times),len(timest0))
\ No newline at end of file
+# print(len(timest0),len(times))
\ No newline at end of file
--
GitLab