Commit 25d7f832 authored by Holger Niemann's avatar Holger Niemann
Browse files

Update to V2.8.0: FLIR data now also downloadable, Port can be now a number or a string

parent 13971823
<<<<<<< HEAD
26.09.2018:
update downloadversionIRdata:
- include FLIR into the downloadscript, Case1 (V2.8.0)
25.09.2018:
update downloadversionIRdata:
- improved performance (V2.7.1), implement switch between test and ArchiveDB (V2.7.0)
13.09.2018:
update downloadversionIRdata:
- bugfixes, verbose level passed now throw, V2.6.1
12.09.2018:
update downloadversionIRdata:
- ToDo list added
- file added to create temperature HDF5 files
21.08.2018:
update downloadversionIRdata:
- remove dependency on readrestdb V2.6.0
- include version number, now 2.5.3 into the files
- bug fixes for downloadversion IRdata
- improvement of exception handling, replace some exception with warnings and improve error messages
......@@ -20,3 +34,15 @@ bugfixes in downloadversionIRdata:
- fixed: stepwise download of data in blocks of 2 seconds (used the wrong indices)
- fixed: not find the infratecfilter
- fixed: wrong time intervall for TC divertor data
Versions:
V3.0.0: download of heat flux data not also possible
V2.8.0: add FLIR camera into the downloadroutine, adapt the port interface, that it accept strings and numbers, numbers are automatically used for QRT
V2.7.1: prove the temperature calculation, less ram usage
V2.7.0: implement dependency on upload_config; switch between ArchiveDB and Test archive possible
V2.6.1: bugfixes with verbose level
V2.6.0: dependency on read_restdb removed
V2.5.3: version number included; improved exception handeling
V2.0.0: upgrade an changes to download OP1.2b data, changes of the method and what is stored
V1.1.0: download of thermocouples implemented
V1.0.0: download test of OP1.2a data
\ No newline at end of file
......@@ -11,15 +11,15 @@ if __name__=='__main__':
import downloadversionIRdata as IR_DL
import numpy as np
localpath=""
time_s=4
time_s=13
programlist = ["20171018.011"]
ports=[51]
programlist = ["20180920.010","20180920.049"]
ports=[31]
for program in programlist:
for port in ports:
try:
exist,dimof,data1,valid=IR_DL.get_temp_from_raw_by_program_V2(portnr=port,program=program,time_s=time_s,threads=4)
exist,dimof,data1,valid=IR_DL.get_temp_from_raw_by_program_V2(portnr=port,program=program,time_s=time_s,threads=4,verbose=5)
if exist:
print("data found")
stream="images"
......@@ -27,10 +27,13 @@ if __name__=='__main__':
data1=data1.swapaxes(0,2)
data1=data1.swapaxes(0,1)
dtype = str(data1.dtype)
tmpfile = localpath+"AEF"+str(port)+"_"+program+"_int_temp_V2"
tmpfile = localpath+"AEF"+str(port)+"_"+program+"_temp_V2"
tmpfile += ".h5"
with h5.File(tmpfile, 'w') as f:
f.create_dataset('timestamps', data=dimof, dtype='uint64' )#,compression="gzip")
f.create_dataset(stream, data=data1, dtype='uint16' , chunks = (np.shape(data1)[0],np.shape(data1)[1],1))
dset=f.create_dataset(stream, shape=np.shape(data1),dtype=dtype , chunks = (np.shape(data1)[0],np.shape(data1)[1],1))
data1=np.ascontiguousarray(data1)
dset.write_direct(data1)
del data1
except Exception as E:
print(E)
\ No newline at end of file
# -*- coding: utf-8 -*-
"""
Created on Tue Jun 5 09:26:06 2018
Version: 2.7.0
Version: 2.8.0
@author: holn
constants and config parameters
"""
......@@ -50,7 +50,8 @@ portcamdict={
'AEF40': 'IRCam_Caleo768kL_0701',
'AEF41': 'IRCam_Caleo768kL_0902',
'AEF50': 'Infratec_9312907',
'AEF51': 'IRCam_Caleo768kL_0905'}
'AEF51': 'IRCam_Caleo768kL_0905',
'AEK51': 'FLIR_SC8303_00037'}
}
camlimdict={
......
......@@ -5,6 +5,7 @@ download:
- implement absolute calibration
- implement download of the stored temperature data (After the upload)
- implement download of the stored heat flux data (After the upload)
- implement download of FLIR data
upload:
- upload the new nucing files for AEF50 (the workaround)
......@@ -12,5 +13,7 @@ upload:
- upload scene models
- upload temperature
- upload heat flux data
- upload FLIR data
- upload FLIR LUT
......@@ -3,8 +3,8 @@
Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018
Version: 2.7.0
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2) . #of updates to add functionalities . #number of updates for bug fixes )
Version: 2.8.0
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
@author: holn
"""
......@@ -726,6 +726,7 @@ if fastDL:
for i in range(threads):
images=images+resultdict[order.index(i)][3]
times=times+resultdict[order.index(i)][2]
resultdict[order.index(i)]=[]
del resultdict
return success,np.array(times),images
# except Exception as E:
......@@ -794,6 +795,94 @@ if fastDL:
images=IR_tools.correct_images(images,badpixels)
out_q.put([threadnumber,successes,time,images])
def download_raw_FLIR_images_via_png(program,time_s,version=0,threads=1,verbose=0):
prog=get_program_from_PID(program)
if not prog[0]:
if verbose>0:
print("program not found, cannot download the data")
return False,[0],[0],0
else:
t1=prog[1]['trigger']['1'][0]
t6=prog[1]['trigger']['6'][0]
larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
if type(time_s)==list:
tstart=time_s[0]
tstop=time_s[1]
if tstop<tstart:
raise Exception("endtime before starttime")
stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
stdate=stdate.isoformat()
enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)
enddate=enddate.isoformat()
else:
stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
stdate=stdate.isoformat()
if time_s==0:
enddate=datetime.datetime.utcfromtimestamp(t6/1e9)
enddate=enddate.isoformat()
else:
enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_s)
enddate=enddate.isoformat()
#"2017-11-15 08:00:00"
typ1_version=get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM")
if typ1_version==None:
mode=1
else:
mode=2
if mode==1:
### okay the data is stored probably as different sets (so recorded with FLIR software)
timess=get_FLIR_Pset_times(stdate,enddate,version)
### we not know the times for all presets for the program, now we have to download them, if times is not empty
if timess==[[],[],[],[]]:
exist=False
time=[]
frames=[]
tsets=[]
else:#timess is not empty, there is data in at least one Pset
exist=True
elif mode==2:##the data is stored only in one datastream, at least this datastream exist
print("not implemented")
else:
raise Exception("cannot identifie the way the searched data is stored!")
return exist,time,frames,mode,tsets
def get_FLIR_Pset_times(stdate,enddate,version=0):
larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
if version==0:
versions=[]
versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS0_DATASTREAM"))
versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS1_DATASTREAM"))
versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS2_DATASTREAM"))
versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS3_DATASTREAM"))
else:
versions=[version,version,version,version]
timess=[]
if version==0:##no version specified search for data in all versions
for version0 in versions:
if version0>0:
found=False
while version0>0 and not found:
try:
times_0=AKF_2.get_time_intervals(larchivepath+"_PS0_DATASTREAM/V"+str(version0),stdate.replace("T"," "),enddate.replace("T"," "))#
found=True
except:
found=False
times_0=[]
version0=version0-1
timess.append(times_0)
else:
timess.append([])
else:
for version0 in versions:
try:
times_0=AKF_2.get_time_intervals(larchivepath+"_PS0_DATASTREAM/V"+str(version0),stdate.replace("T"," "),enddate.replace("T"," "))#
except:
times_0=[]
timess.append(times_0)
return timess
def download_last_raw_image_by_time(port,starttime,stoptime,version=0):
OP=get_OP_by_time(starttime)
Cam=portcamdict[OP]['AEF'+str(port)]
......@@ -1293,6 +1382,30 @@ def get_temp_from_raw_by_program_V2(portnr,program,time_s=0,emi=0.8,version=0,th
return get_temp_from_raw_by_program(portnr,program,time_s=time_s,emi=emi,T_version=2,version=version,threads=threads,give_ERROR=give_ERROR,use_firstframe_as_background=use_firstframe_as_background)
def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,back_emissivity=0.8,verbose=0):
FLIR=False
if type(portnr) is str:
try:
camera=portcamdict[portnr]
if camera.split("_")[0]=='FLIR':
FLIR=True
else:
FLIR=False
portnr=int(portnr.split("AEF")[1])
goon=True
except Exception as E:
raise Exception("unknown Port!"+E)
elif type(portnr) is int:
FLIR=False
goon=True
else:
goon=False
if not goon:
raise Exception("the given port is neither a number or a valid String!")
else:
if FLIR:
print("to be implemented")
exist,time,frames,mode=download_raw_FLIR_images_via_png(program,time_s,version,threads,verbose=verbose-1)
else:
cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version,back_emissivity,verbose=verbose-1)
if not cexist:
if verbose>0:
......@@ -1323,6 +1436,7 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
success,frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
else:
success,frames=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
print(datetime.datetime.now(),"Temperature calculation done")
if not success:
if verbose>0:
print("calculation of the temperature failed, calibration could not be applied")
......@@ -1338,11 +1452,11 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
print(datetime.datetime.now(),"checking temperaturerange")
valid=True
# for i in range(len(frames)):
try:
check_temperature_range(time[0],np.max(frames)-273.15,np.min(frames[:][np.nonzero(frames[i])])-273.15,portnr,t_exp,cfilter)
except Exception as E:
print(E)
valid=False
# try:
# check_temperature_range(time[0],np.max(frames)-273.15,np.min(frames[:][np.nonzero(frames[i])])-273.15,portnr,t_exp,cfilter)
# except Exception as E:
# print(E)
# valid=False
# i=len(frames)
date=int(program.split(".")[0])
if date>20180712 and date<20180720:
......@@ -1603,14 +1717,14 @@ def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_erro
if verbose>0:
print(datetime.datetime.now(),"background treatment")
# if fullbackground:
#sbackground=np.zeros(np.shape(images[0]),dtype=np.float32)+background
for i in range(len(images)):
images[i]=(images[i]-background).astype(np.float32)
images[i]=np.clip((images[i]-background),a_min=0,a_max=None).astype(np.uint16)
# else:
# for i in range(len(images)):
# images[i]=images[i]-background
# images=np.array(images.clip(min=0),dtype=np.uint16)
images=[np.round(im.clip(min=0)).astype(np.uint16) for im in images]
# images=[np.round(im.clip(min=0)).astype(np.uint16) for im in images]
if verbose>0:
print(datetime.datetime.now(),"applying LUT")
LUT=np.array([LUT[1],LUT[2]])
......@@ -1635,6 +1749,51 @@ def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_erro
raise Warning(E)
return False,[0]
#def apply_calib_on_raw_fast(images,background,LUT,refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,fullbackground=False,give_ERROR=False, verbose=0):
# try:
# LUT=np.array([LUT[1],LUT[2]])
# if type(gain)!=int and type(offset)!=int:
# if verbose>0:
# print(datetime.datetime.now(),"NUCing")
# # eliminate bad offset and gain points
# offset[offset==np.inf] = 0
# offset[offset==-np.inf] = 0
# gain[gain==np.inf] = 1
# gain[gain==-np.inf] = 1
# if give_ERROR:
# terror=[]
# for i in range(len(images)):
# images[i]=(images[i]*gain+offset).astype(np.float32)
# terror=terror+[terror_from_LUT(LUT,images[i])]
# eima=np.round(images[i]*gain_error + offset_error).astype(np.uint16)
# max_image = temp_from_LUT(LUT,images[i]+eima)
# min_image = temp_from_LUT(LUT,images[i]-eima)
# terror[i] = (terror[i] + (max_image - min_image)/2).astype(np.float32)
# return True,images,terror
# else:
# for i in range(len(images)):
# images[i]=(temp_from_LUT(LUT,np.clip(images[i]*gain+offset-background,a_min=0,a_max=None).astype(np.uint16))+(refT+273.15)).astype(np.float32)
# return True,images
# else:
# if verbose>0:
# print(datetime.datetime.now(),"start Temperature calculation")
# if give_ERROR:
# terror=[]
# for i in range(len(images)):
# images[i]=(temp_from_LUT(LUT,np.clip(images[i]-background,a_min=0,a_max=None).astype(np.uint16))+(refT+273.15)).astype(np.float32)
# terror=terror+[terror_from_LUT(LUT,images[i])]
# max_image = temp_from_LUT(LUT,images[i])
# min_image = temp_from_LUT(LUT,images[i])
# terror[i] = (terror[i] + (max_image - min_image)/2).astype(np.float32)
# return True,images,terror
# else:
# for i in range(len(images)):
# images[i]=(temp_from_LUT(LUT,np.clip(images[i]-background,a_min=0,a_max=None).astype(np.uint16))+(refT+273.15)).astype(np.float32)
# return True,images
# except Exception as E:
# raise Warning(E)
# return False,[0]
def apply_NUC(images,gain,offset,gain_error=0,offset_error=0,give_error=False):
"""
apply_NUC(images=numpy array(time,width,height),gain,offset =numpy array(width,height) )
......@@ -2088,124 +2247,48 @@ def get_program_from_PID(instring):
end of the copied functions
@author: thomsen
"""
#%%
#%% dummy run of the script
if __name__=='__main__':
print("local function calling")
port=10
# prog="20171108.015"
prog="20171109.021"
# prog="20171025.030"
# prog="20171012.035"
prog="20171005.024"
prog="20171109.056"
prog="20171207.050"
program="20180712.027"
port=20
prog="20180724.030"
port=51
program="20180809.010"
# port=21
# prog="20180822.017"
# status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[6,7],emi=0.82,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
# plt.figure()
# for port in [10,11,20,21,30,31,40,41,51]:
# framerate=get_camera_framerate_by_program(port,program)
# if framerate[0]:
# plt.plot(framerate[1],framerate[2],label="AEF"+str(port))
# plt.legend()
# plt.grid()
port=10
prog="20180816.016"
prog="20180816.018"#[0,6]
port=21
prog="20180822.017"
status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[6,7],emi=0.82,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
# status,time,images=download_raw_images_by_program_via_png(port,prog,time_s=1,threads=4,verbose=0)
# test=get_NUC_by_program(port,prog,8,version=0,verbose=5)
# print(len(images))
# prog=get_program_from_PID(prog)
# if prog[0]:
# t0time=prog[1]['trigger']['0'][0]
# starttime=prog[1]['trigger']['1'][0]
# import h5py as h5
# szene=h5.File("X:\\E4 Diagnostics\\QIR\Calibrations\\scene_models\\AEF"+str(port)+"\\AEF"+str(port)+"_scene_model.h5")
# szene=h5.File("\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\Software\\QI-RealTime\\1.1.0\\QRT-Analysis\\release\\Config\\spatial_calibration\\AEF"+str(port)+"_scene_model.h5",'r')
# szene=h5.File("X:\\E4 Diagnostics\\QIR\Data\\For Holger\\AEF21_scene_model_1531872000000000000.h5",'r')
# Xloc=np.asarray(szene['x'])
# Yloc=np.asarray(szene['y'])
# Zloc=np.asarray(szene['z'])
# CAD=np.asarray(szene['CAD'])
# timpo=0.87
# fid=IR_tools.find_nearest((np.asarray(time)-time[0])/1e9,timpo)
# plt.figure()#
# plt.imshow(CAD,cmap='gray')
# plt.imshow(images[fid-8]-images[fid-28],vmin=0,vmax=60,cmap=exJet_trans)
# cb=plt.colorbar()
# cb.set_label("temperature difference K", rotation=270,labelpad=26,fontsize=26)
# cb.ax.tick_params(labelsize=26)
# plt.figure()#
# plt.imshow(images[fid-20]-images[fid-40],vmin=0,vmax=40,cmap=exJet)
# plt.imshow(images[-1],vmin=330,vmax=1000,cmap=exJet)
# cb=plt.colorbar()
# cb.set_label("temperature difference K", rotation=270,labelpad=26,fontsize=26)
# cb.ax.tick_params(labelsize=26)
# plt.figure()# # plt.imshow(CAD,cmap='gray')
# plt.imshow(images[fid-40],vmin=300,vmax=400,cmap=exJet)
# cb=plt.colorbar()
# cb.set_label("temperature K", rotation=270,labelpad=26,fontsize=26)
# cb.ax.tick_params(labelsize=26)
# proglist=["20180801.014","20180801.015","20180801.016","20180801.018","20180801.019"]
# portlist=[20,21,51]
# version=0
# time_s=30
# for program in proglist:
# for port in portlist:
# prog=AKF_1.get_program_from_PID(program)
# if prog[0]:
# t0time=prog[1]['trigger']['0'][0]
# starttime=prog[1]['trigger']['1'][0]
# try:
# stoptime=prog[1]['trigger']['6'][0]
# except:
# stoptime=prog[1]['upto']
# success=True
# OP=get_OP_by_time(starttime)
# Cam=portcamdict[OP]['AEF'+str(port)]
# if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
# if version==0:
# version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
# larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
# elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
# if version==0:
# version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
# larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
# else:
# raise Exception("Port number does not fit the known cameras")
# if type(time_s)==list:
# tstart=time_s[0]
# tstop=time_s[1]
# if tstop<tstart:
# raise Exception("endtime before starttime")
# stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
# stdate=stdate.isoformat()
# enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
# enddate=enddate.isoformat()
# else:
# stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
# stdate=stdate.isoformat()
# t0date=datetime.datetime.utcfromtimestamp((t0time-100)/1e9)
# t0date=t0date.isoformat()
# if time_s==0:
# enddate=datetime.datetime.utcfromtimestamp(stoptime+100/1e9)
# enddate=enddate.isoformat()
# else:
# enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)
# enddate=enddate.isoformat()
# #"2017-11-15 08:00:00"
# try:
# times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
# except:
# times=[]
# try:
# timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),stdate.replace("T"," "))#
# except:
# timest0=[]
# print(program,"AEF"+str(port),len(timest0),len(times))
\ No newline at end of file
# cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20)
# cb.ax.tick_params(labelsize=20)
port=41
program="20180925.013"
prog=get_program_from_PID(program)
if prog[0]:
t0=prog[1]['trigger']['0'][0]
t1=prog[1]['trigger']['1'][0]
t6=prog[1]['trigger']['6'][0]
for port in [10]:
# bla=download_raw_images_by_times(port,t0,t1,verbose=5)
bla=get_temp_from_raw_by_program(port,program,time_s=[0,4],emi=0.82,T_version=1,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=True,verbose=5)
import h5py as h5
File=h5.File("AEF"+str(port)+"_"+program+"_temp_v1_b.h5",'w')
imags=np.asarray(bla[2]).swapaxes(0,2).swapaxes(0,1)
shapi=np.shape(imags)
dset=File.create_dataset('images',shape=(shapi[0],shapi[1],shapi[2]),dtype='uint16',chunks=(shapi[0],shapi[1],1))
imags=np.ascontiguousarray(imags)
dset.write_direct(imags)
File.create_dataset('timestamps', data=list(bla[1]), dtype='uint64')#,compression="gzip")
File.close()
# prog="20171109.045"
# for port in [10,11,20,21,30,31,40,41,50,51]:
# points=valid_background_rectangle[port]
# status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,0.1],emi=0.82,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
# plt.figure()
# plt.imshow(images[0],vmin=330,vmax=600)
# plt.savefig("AEF"+str(port)+"_ori.png")
# plt.close()
# dummy=images[0]#[points[1]:points[3],points[0]:points[2]]#FOV*image#
# dummy[points[1]:points[3],points[0]:points[2]]=1500
# plt.figure()
# plt.imshow(dummy,vmin=330,vmax=600)
# plt.savefig("AEF"+str(port)+"_boxed.png")
# plt.close()
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment