diff --git a/CHANGELOG b/CHANGELOG
index 6e421102420305b87f1871fb535b879540fbf25e..15e9d385c119c6dc33da666cb2ee0227b40522a4 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,20 @@
<<<<<<< HEAD
+26.09.2018:
+update downloadversionIRdata:
+ - include FLIR into the downloadscript, Case1 (V2.8.0)
+25.09.2018:
+update downloadversionIRdata:
+ - improved performance (V2.7.1), implement switch between test and ArchiveDB (V2.7.0)
+13.09.2018:
+update downloadversionIRdata:
+ - bugfixes, verbose level passed now throw, V2.6.1
+12.09.2018:
+update downloadversionIRdata:
+ - ToDo list added
+ - file added to create temperature HDF5 files
21.08.2018:
update downloadversionIRdata:
+ - remove dependency on readrestdb V2.6.0
- include version number, now 2.5.3 into the files
- bug fixes for downloadversion IRdata
- improvement of exception handling, replace some exception with warnings and improve error messages
@@ -19,4 +33,16 @@ changes in downloadversionIRdata:
bugfixes in downloadversionIRdata:
- fixed: stepwise download of data in blocks of 2 seconds (used the wrong indices)
- fixed: not find the infratecfilter
- - fixed: wrong time intervall for TC divertor data
\ No newline at end of file
+ - fixed: wrong time intervall for TC divertor data
+
+Versions:
+V3.0.0: download of heat flux data not also possible
+V2.8.0: add FLIR camera into the downloadroutine, adapt the port interface, that it accept strings and numbers, numbers are automatically used for QRT
+V2.7.1: prove the temperature calculation, less ram usage
+V2.7.0: implement dependency on upload_config; switch between ArchiveDB and Test archive possible
+V2.6.1: bugfixes with verbose level
+V2.6.0: dependency on read_restdb removed
+V2.5.3: version number included; improved exception handeling
+V2.0.0: upgrade an changes to download OP1.2b data, changes of the method and what is stored
+V1.1.0: download of thermocouples implemented
+V1.0.0: download test of OP1.2a data
\ No newline at end of file
diff --git a/Create_HDF5_from_archive_data.py b/Create_HDF5_from_archive_data.py
index 2ebaaffde89b9d50275582f987af5eeaa72ddfec..2278cb52939e75350f4370583f9f0e7148b3ca8e 100644
--- a/Create_HDF5_from_archive_data.py
+++ b/Create_HDF5_from_archive_data.py
@@ -11,15 +11,15 @@ if __name__=='__main__':
import downloadversionIRdata as IR_DL
import numpy as np
localpath=""
- time_s=4
+ time_s=13
- programlist = ["20171018.011"]
- ports=[51]
+ programlist = ["20180920.010","20180920.049"]
+ ports=[31]
for program in programlist:
for port in ports:
try:
- exist,dimof,data1,valid=IR_DL.get_temp_from_raw_by_program_V2(portnr=port,program=program,time_s=time_s,threads=4)
+ exist,dimof,data1,valid=IR_DL.get_temp_from_raw_by_program(portnr=port,program=program,time_s=time_s,threads=4,verbose=5)
if exist:
print("data found")
stream="images"
@@ -27,10 +27,13 @@ if __name__=='__main__':
data1=data1.swapaxes(0,2)
data1=data1.swapaxes(0,1)
dtype = str(data1.dtype)
- tmpfile = localpath+"AEF"+str(port)+"_"+program+"_int_temp_V2"
+ tmpfile = localpath+"AEF"+str(port)+"_"+program+"_temp_V2"
tmpfile += ".h5"
with h5.File(tmpfile, 'w') as f:
f.create_dataset('timestamps', data=dimof, dtype='uint64' )#,compression="gzip")
- f.create_dataset(stream, data=data1, dtype='uint16' , chunks = (np.shape(data1)[0],np.shape(data1)[1],1))
+ dset=f.create_dataset(stream, shape=np.shape(data1),dtype=dtype , chunks = (np.shape(data1)[0],np.shape(data1)[1],1))
+ data1=np.ascontiguousarray(data1)
+ dset.write_direct(data1)
+ del data1
except Exception as E:
print(E)
\ No newline at end of file
diff --git a/IR_config_constants.py b/IR_config_constants.py
index 73d42c191bdb5ac702b38985f161b653180998a0..e25851093ab9a46b89bce1b254be1eb03e7f45ca 100644
--- a/IR_config_constants.py
+++ b/IR_config_constants.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
Created on Tue Jun 5 09:26:06 2018
-Version: 2.7.0
+Version: 2.8.0
@author: holn
constants and config parameters
"""
@@ -50,7 +50,8 @@ portcamdict={
'AEF40': 'IRCam_Caleo768kL_0701',
'AEF41': 'IRCam_Caleo768kL_0902',
'AEF50': 'Infratec_9312907',
- 'AEF51': 'IRCam_Caleo768kL_0905'}
+ 'AEF51': 'IRCam_Caleo768kL_0905',
+ 'AEK51': 'FLIR_SC8303_00037'}
}
camlimdict={
diff --git a/IR_image_tools.py b/IR_image_tools.py
index 9df7318d4eab758ee4ff7b050bd6487d4b8719c2..539109c9d94e79e0720fdeace0fecded9d56d1bc 100644
--- a/IR_image_tools.py
+++ b/IR_image_tools.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
Created on Wed May 9 14:56:32 2018
-Version: 2.5.3
+Version: 2.8.0
@author: Holger Niemann, Peter Drewelow, Yu Gao
mainly to clean up the downloadversionIRdata code
@@ -20,7 +20,6 @@ from os.path import join, basename
import glob
import datetime
-
def get_OP_by_time(time_ns=None, shot_no=None, program_str=None):
'''Derives operation phase (OP) of W7-X based on either:
a nanosacond time stamp, a MDSplus style shot no. or a program ID.
@@ -62,7 +61,6 @@ def get_OP_by_time(time_ns=None, shot_no=None, program_str=None):
else:
return None
-
def bestimmtheitsmass_general(data,fit):
R=0
if len(fit)==len(data):
@@ -222,7 +220,6 @@ def find_outlier_pixels(frame,tolerance=3,worry_about_edges=True,plot_it=False):
return bad_pixels
-
def correct_images(images,badpixels):
print('correct_images: New routine restore_bad_pixels() is used and can be called directly. Check out "help(restore_bad_pixels)"')
if type(badpixels)!=int:
@@ -479,4 +476,20 @@ def load_ref_images(port, exposuretime):
def reconstruct_coldframe (exposuretime, sT, a, bnew, coldref):
cirebuild = a * sT + bnew * exposuretime + coldref
- return cirebuild
\ No newline at end of file
+ return cirebuild
+
+
+#%% other functions
+def check_dublicates(array):
+ a = array
+ import collections
+ return [item for item, count in collections.Counter(a).items() if count > 1]
+
+def check_dublicates_2(array):
+ seen = set()
+ uniq = []
+ for x in array:
+ if x not in seen:
+ uniq.append(x)
+ seen.add(x)
+ return uniq,seen
\ No newline at end of file
diff --git a/ToDO.txt b/ToDO.txt
index 99d91504fd1372e061aff65ba4ddf287d1d771ce..63c37a94dc87b352959388933030d621f9061c01 100644
--- a/ToDO.txt
+++ b/ToDO.txt
@@ -5,6 +5,7 @@ download:
- implement absolute calibration
- implement download of the stored temperature data (After the upload)
- implement download of the stored heat flux data (After the upload)
+- implement download of FLIR data
upload:
- upload the new nucing files for AEF50 (the workaround)
@@ -12,5 +13,7 @@ upload:
- upload scene models
- upload temperature
- upload heat flux data
+- upload FLIR data
+- upload FLIR LUT
diff --git a/downloadversionIRdata.py b/downloadversionIRdata.py
index 2fe655e0bd050afe7da44dfb931504294044340b..34998aa5d279e85f3701100c1a598b7f36db88d3 100644
--- a/downloadversionIRdata.py
+++ b/downloadversionIRdata.py
@@ -3,8 +3,8 @@
Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018
-Version: 2.7.0
-(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2) . #of updates to add functionalities . #number of updates for bug fixes )
+Version: 2.8.0
+(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
@author: holn
"""
@@ -636,24 +636,27 @@ if fastDL:
else:
starttime=prog[1]['trigger']['1'][0]
stoptime=prog[1]['trigger']['6'][0]
- success=True
OP=get_OP_by_time(starttime)
Cam=portcamdict[OP]['AEF'+str(port)]
if OP=="OP1.2a":
if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
if version==0:
version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
- larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+ larchivepath=archivepath+"Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
if version==0:
version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
- larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+ larchivepath=archivepath+"W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
else:
raise Exception("Port number does not fit the known cameras")
elif OP=="OP1.2b":
if version==0:
version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
- larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+ larchivepath=archivepath+"W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+ return download_raw_images_by_time_via_png(larchivepath,starttime,stoptime,time_s=time_s,threads=threads,version=version,verbose=verbose-1)
+
+ def download_raw_images_by_time_via_png(larchivepath,starttime,stoptime,time_s=0,threads=1,version=0,verbose=0):
+ success=True
if type(time_s)==list:
tstart=time_s[0]
tstop=time_s[1]
@@ -676,6 +679,7 @@ if fastDL:
try:
times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
except Exception as E:
+ print(larchivepath)
raise Warning(E)
return False,[0],[0]
time=[]
@@ -684,7 +688,7 @@ if fastDL:
if threads==1:
for i in range(lnt):
ele=times[lnt-1-i]
- imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
+ imag=download_last_raw_image_by_time(larchivepath,ele[0]-10,ele[0]+10)
if imag[0]:
time.append(ele[0])
images.append(np.array(imag[1],dtype=np.uint16))
@@ -705,7 +709,7 @@ if fastDL:
for i in range(threads):
if verbose>0:
print("Start Thread ",i+1)
- p = multiprocessing.Process(target=download_raw_images_png_by_times_thread, args=(port,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,))
+ p = multiprocessing.Process(target=download_raw_images_png_by_times_thread, args=(larchivepath,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,))
jobs.append(p)
p.start()
resultdict = []
@@ -726,18 +730,19 @@ if fastDL:
for i in range(threads):
images=images+resultdict[order.index(i)][3]
times=times+resultdict[order.index(i)][2]
+ resultdict[order.index(i)]=[]
del resultdict
return success,np.array(times),images
# except Exception as E:
# raise Exception(E)
- def download_raw_images_png_by_times_thread(port,times,out_q,threadnumber,version=0):
+ def download_raw_images_png_by_times_thread(larchivepath,times,out_q,threadnumber,version=0):
images=[]
time=[]
successes=[]
for i in times:
- imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
+ imag=download_last_raw_image_by_time(larchivepath,i-10,i+10,version=version)
if imag[0]:
images.append(np.array(imag[1],dtype=np.uint16))
# print(np.min(imag[1]),np.max(imag[1]))
@@ -748,12 +753,12 @@ if fastDL:
out_q.put([threadnumber,successes,time,images])
# return success,np.array(time),np.array(images,dtype=np.uint16)
- def convert_raw_to_temp_thread(port,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
+ def convert_raw_to_temp_thread(larchivepath,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
images=[]
time=[]
- successes=[]
+ successes=[]
for i in times:
- imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
+ imag=download_last_raw_image_by_time(larchivepath,i-10,i+10,version=version)
if imag[0]:
images.append(np.array(imag[1],dtype=np.uint16))
time.append(i)
@@ -771,12 +776,12 @@ if fastDL:
images=IR_tools.correct_images(images,badpixels)
out_q.put([threadnumber,successes,time,images])
- def NUC_raw_thread(port,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
+ def NUC_raw_thread(larchivepath,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
images=[]
time=[]
successes=[]
for i in times:
- imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
+ imag=download_last_raw_image_by_time(larchivepath,i-10,i+10,version=version)
if imag[0]:
images.append(np.array(imag[1],dtype=np.uint16))
time.append(i)
@@ -793,20 +798,200 @@ if fastDL:
print(datetime.datetime.now(),"correcting bad pixels")
images=IR_tools.correct_images(images,badpixels)
out_q.put([threadnumber,successes,time,images])
-
-def download_last_raw_image_by_time(port,starttime,stoptime,version=0):
- OP=get_OP_by_time(starttime)
- Cam=portcamdict[OP]['AEF'+str(port)]
- if Cam.split("_")[0]=="Infratec" and OP=="OP1.2a":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
- if version==0:
- version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
- larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
- elif Cam.split("_")[0]=="IRCam"or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
- if version==0:
- version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
- larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+
+def download_raw_FLIR_images_via_png(t1,t6,time_s=0,version=0,threads=1,verbose=0):
+ larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
+ if type(time_s)==list:
+ tstart=time_s[0]
+ tstop=time_s[1]
+ if tstop<tstart:
+ raise Exception("endtime before starttime")
+ stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
+ stdate=stdate.isoformat()
+ enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)
+ enddate=enddate.isoformat()
+ else:
+ stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
+ stdate=stdate.isoformat()
+ if time_s==0:
+ enddate=datetime.datetime.utcfromtimestamp(t6/1e9)
+ enddate=enddate.isoformat()
+ else:
+ enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_s)
+ enddate=enddate.isoformat()
+ #"2017-11-15 08:00:00"
+ typ1_version=get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM")
+ if typ1_version==None:
+ mode=1
+ else:
+ mode=2
+ if mode==1:
+ ### okay the data is stored probably as different sets (so recorded with FLIR software)
+ timess,versions=get_FLIR_Pset_times(stdate,enddate,version)
+ ### we not know the times for all presets for the program, now we have to download them, if times is not empty
+ exist,time,frames,tsets=download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=verbose-1)
+ elif mode==2:##the data is stored only in one datastream, at least this datastream exist
+ raise Exception("not implemented")
+# time=[]
+# images=[]
+# lnt=len(times)
+# if threads==1:
+# for i in range(lnt):
+# ele=times[lnt-1-i]
+# imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
+# if imag[0]:
+# time.append(ele[0])
+# images.append(np.array(imag[1],dtype=np.uint16))
+# else:
+# success=False
+# return success,np.array(time),images
else:
- raise Exception("camera unknown, stopping here")
+ raise Exception("cannot identifie the way the searched data is stored!")
+
+ return exist,time,frames,mode,tsets,versions
+
+def get_FLIR_Pset_times(stdate,enddate,version=0):
+ larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
+ if version==0:
+ versions=[]
+ versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS0_DATASTREAM"))
+ versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS1_DATASTREAM"))
+ versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS2_DATASTREAM"))
+ versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS3_DATASTREAM"))
+ else:
+ versions=[version,version,version,version]
+ timess=[]
+ if version==0:##no version specified search for data in all versions
+ counter=-1
+ for version0 in versions:
+ counter+=1
+ if version0>0:
+ found=False
+ while (version0>0 and not found):
+ try:
+ times_0=AKF_2.get_time_intervals(larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw",stdate.replace("T"," "),enddate.replace("T"," "))#
+ found=True
+ except Exception as E:
+ print(E)
+ found=False
+ times_0=[]
+ version0=version0-1
+ timess.append(times_0)
+ else:
+ timess.append([])
+ else:
+ counter=-1
+ for version0 in versions:
+ counter+=1
+ try:
+ times_0=AKF_2.get_time_intervals(larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw",stdate.replace("T"," "),enddate.replace("T"," "))#
+ except:
+ times_0=[]
+ timess.append(times_0)
+ return timess,versions
+
+def download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=0):
+ if len(timess[0])==len(timess[1]) and len(timess[2])==len(timess[3]) and len(timess[0])==0:
+ exist=False
+ time=[]
+ frames=[]
+ tsets=[]
+ else:#timess is not empty, there is data in at least one Pset
+ if verbose>0:
+ print(datetime.datetime.now(),"data found, start the download")
+ exist=True
+ timesets=[]
+ framess=[]
+ success=True
+ Psets=[]
+ if threads==1:
+ counter=-1
+ setn=-1
+ for times in timess:
+ setn+=1
+ if times!=[]:
+ counter+=1
+ Psets.append(counter)
+ framess.append([])
+ timesets.append([])
+ lnt=len(times)
+ for i in range(lnt):
+ ele=times[lnt-1-i]
+ imag=download_last_raw_image_by_time(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw",ele[0]-10,ele[0]+10)
+ if imag[0]:
+ timesets[counter].append([ele[0],i,setn,counter])
+ framess[counter].append(np.array(imag[1],dtype=np.uint16))
+ else:
+ success=False
+ else:#several threads, each for a preset
+ tim=[]
+ counter=-1
+ setn=-1
+ threads=0
+ larchivepaths=[]
+ versionss=[]
+ for times in timess:
+ setn+=1
+ if times!=[]:
+ counter+=1
+ threads+=1
+ Psets.append(counter)
+ larchivepaths.append(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw")
+ tim.append(times)
+ versionss.append(versions[setn])
+ jobs = []
+ out_q=multiprocessing.Queue()
+ for i in range(threads):
+ if verbose>0:
+ print("Start Thread ",i+1)
+ p = multiprocessing.Process(target=download_raw_images_png_by_times_thread, args=(larchivepaths[i],tim[i],out_q,i,versionss[i],))
+ jobs.append(p)
+ p.start()
+ resultdict = []
+ for i in range(threads):
+ resultdict.append(out_q.get())
+ for p in jobs:
+ p.join()
+ if verbose>0:
+ print("all threads are done")
+ order=[]
+ for ele in resultdict:
+ order.append(ele[0])
+ if len(np.where(np.asarray(ele[1])==False)[0])>0:
+ success=False
+ for i in range(threads):
+ framess.append(resultdict[order.index(i)][3])
+ timeline=[]
+ for j in range(len(resultdict[order.index(i)][2])):
+ timeline.append([resultdict[order.index(i)][2][j],j,Psets[i],i])
+ timesets.append(timeline)
+ resultdict[order.index(i)]=[]
+ del resultdict,timeline
+ if verbose>0:
+ print(datetime.datetime.now(),"download finished, successfull",success)
+ if success:
+ del timess
+ ### okay the have now the data from all presets, now we have to sort the frames into one array
+ time=[]
+ frames=[]
+ dummyT=[]
+ tsets=[]
+ for tim in timesets:
+ dummyT=dummyT+tim
+ dummyT.sort()
+ for ele in dummyT:
+ time.append(ele[0])
+ frames.append(framess[ele[2]][ele[1]])
+ tsets.append(ele[2])
+ del timesets,framess
+ else:
+ exist=False
+ time=[]
+ frames=[]
+ tsets=[]
+ return exist,time,frames,tsets
+
+def download_last_raw_image_by_time(larchivepath,starttime,stoptime,version=0):
try:
res = urllib.request.urlopen(larchivepath+"/_signal.png?from="+str(starttime-10)+"&upto="+str(stoptime))
img = Image.open(BytesIO(res.read()))
@@ -1126,8 +1311,7 @@ def get_TC_data_for_port_by_program(port,program,part=0,version=0):
def get_TC_data_for_port(port,starttime,endtime,part=0,version=0):
urlstart="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.115/DataModuleDesc.19471_DATASTREAM/"
- #66,67(10),72,73(20),78,79(30),84,85(40/50_2),90,91(50_1)
-
+ #66,67(10),72,73(20),78,79(30),84,85(40/50_2),90,91(50_1)
n=0
success=True
signal=0
@@ -1293,71 +1477,149 @@ def get_temp_from_raw_by_program_V2(portnr,program,time_s=0,emi=0.8,version=0,th
return get_temp_from_raw_by_program(portnr,program,time_s=time_s,emi=emi,T_version=2,version=version,threads=threads,give_ERROR=give_ERROR,use_firstframe_as_background=use_firstframe_as_background)
def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,back_emissivity=0.8,verbose=0):
- cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version,back_emissivity,verbose=verbose-1)
- if not cexist:
- if verbose>0:
- print("Unable to load the complete calibration data for",program," please confirm that this data was uploaded")
- return False,[0],[0],False
- else:
- if verbose>0:
- print(datetime.datetime.now(),"raw download start")
- if fastDL:
- exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_s,version,threads,verbose=verbose-1)
+ FLIR=False
+ prog=get_program_from_PID(program)
+ if prog[0]:
+ t0=prog[1]['trigger']['0'][0]
+ starttime=prog[1]['trigger']['1'][0]
+ stoptime=prog[1]['trigger']['6'][0]
+ success=True
+ OP=get_OP_by_time(starttime)
+
+ if type(portnr) is str:
+ try:
+ camera=portcamdict[OP][portnr]
+ if camera.split("_")[0]=='FLIR':
+ FLIR=True
+ else:
+ FLIR=False
+ portnr=int(portnr.split("AEF")[1])
+ goon=True
+ except Exception as E:
+ raise Exception("unknown Port!"+E)
+ elif type(portnr) is int:
+ FLIR=False
+ goon=True
else:
- exist,time,frames=download_raw_images_by_program(portnr,program,time_s,version,verbose=verbose-1)
- if exist:
- frames=[im.swapaxes(0,1) for im in frames]
- else:
- raise Exception("no data found")
- if not exist:
- raise Warning("data not found in database!")
- return False,[0],[0],False
+ goon=False
+ if not goon:
+ raise Exception("the given port is neither a number or a valid String!")
else:
- if verbose>0:
- print(datetime.datetime.now(),"raw download finished")
- FOV=get_FOV_mask(portnr)
- if use_firstframe_as_background:
- bim=apply_NUC([frames[0]],gain,offset)
- background=get_average_background_recangle(portnr,bim[0])
- if give_ERROR:
- success,frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
- else:
- success,frames=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
- if not success:
- if verbose>0:
- print("calculation of the temperature failed, calibration could not be applied")
- return False,time,frames,False
+ if FLIR:
+ print("to be implemented")
+ exist,time,frames,mode,tsets,versions=download_raw_FLIR_images_via_png(t1=starttime,t6=stoptime,time_s=time_s,version=version,threads=threads,verbose=verbose-1)
+ if exist:
+ if mode==1:
+ Psets=IR_tools.check_dublicates(tsets)
+ Radi_Co=[]
+ Temp_Co=[]
+ for PS in range(4):
+ if PS in Psets:
+ parlog=read_restdb_old(archivepath+"W7X/QSR07_FLIR/AEK51_raw_PS"+str(PS)+"_PARLOG/V"+str(versions[PS])+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime+20))
+ if parlog[0]:
+ pari=parlog[2][0]['meta-data']
+ c1=[float(pari['Coeff0_'+str(PS)].split("\n")[0]),float(pari['Coeff1_'+str(PS)].split("\n")[0])]
+ c2=[float(pari['TempCoeff'+str(x)+'_'+str(PS)].split("\n")[0]) for x in range(7)]
+ Radi_Co.append(c1)
+ Temp_Co.append(c2)
+ else:
+ Radi_Co.append([])
+ Temp_Co.append([])
+ else:
+ Radi_Co.append([])
+ Temp_Co.append([])
+ ### got the raw, knowing which frame needs which calibration, got the calibration, not we have to transform it
+ if verbose>0:
+ print(datetime.datetime.now(),"Converting the raw data into temperature, number of frames:",len(frames))
+ for i in range(len(frames)):
+ try:
+ frames[i]=Radi_Co[tsets[i]][0]+frames[i]*Radi_Co[tsets[i]][1]
+ dummy=frames[i]
+ temp=Temp_Co[tsets[i]][0]+Temp_Co[tsets[i]][1]*dummy
+ for expo in range(2,7):
+ temp=temp+Temp_Co[tsets[i]][expo]*np.power(dummy,expo)
+ frames[i]=temp+273.15
+ except Exception as E:
+ print(E,i,len(tsets),Radi_Co[tsets[i]],Temp_Co[tsets[i]])
+ if verbose>0:
+ print(datetime.datetime.now(),"temperature frames done")
+ return exist,time,frames,tsets
+ elif mode==2:
+ raise Exception("not implemented")
+ else:
+ raise Exception("FLIR evaluation, This case should not be possible")
+ else:
+ print("FLIR data not found")
+ return False,[0],[0],[]
else:
-
- for i in range(len(frames)):
- frames[i]=(frames[i]*FOV).astype(np.float32)
- if verbose>0:
- print(datetime.datetime.now(),"correcting bad pixels")
- frames=IR_tools.correct_images(frames,badpixels)
- if verbose>0:
- print(datetime.datetime.now(),"checking temperaturerange")
- valid=True
- # for i in range(len(frames)):
- try:
- check_temperature_range(time[0],np.max(frames)-273.15,np.min(frames[:][np.nonzero(frames[i])])-273.15,portnr,t_exp,cfilter)
- except Exception as E:
- print(E)
- valid=False
- # i=len(frames)
- date=int(program.split(".")[0])
- if date>20180712 and date<20180720:
- prog=get_program_from_PID(program)
- if prog[0]:
- # t0time=prog[1]['trigger']['0'][0]
- starttime=prog[1]['trigger']['1'][0]
- toff=time[0]-starttime
- if toff>10e6:
- print("time offset detected, offset is "+str(toff)+" ns, correcting")
- time[:]=time[:]-toff
- if give_ERROR:
- return exist,time,frames,valid,error_images
+ cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version,back_emissivity,verbose=verbose-1)
+ if not cexist:
+ if verbose>0:
+ print("Unable to load the complete calibration data for",program," please confirm that this data was uploaded")
+ return False,[0],[0],False
else:
- return exist,time,frames,valid
+ if verbose>0:
+ print(datetime.datetime.now(),"raw download start")
+ if fastDL:
+ exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_s,version,threads,verbose=verbose-1)
+ else:
+ exist,time,frames=download_raw_images_by_program(portnr,program,time_s,version,verbose=verbose-1)
+ if exist:
+ frames=[im.swapaxes(0,1) for im in frames]
+ else:
+ raise Exception("no data found")
+ if not exist:
+ raise Warning("data not found in database!")
+ return False,[0],[0],False
+ else:
+ if verbose>0:
+ print(datetime.datetime.now(),"raw download finished")
+ FOV=get_FOV_mask(portnr)
+ if use_firstframe_as_background:
+ bim=apply_NUC([frames[0]],gain,offset)
+ background=get_average_background_recangle(portnr,bim[0])
+ if give_ERROR:
+ success,frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
+ else:
+ success,frames=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
+ print(datetime.datetime.now(),"Temperature calculation done")
+ if not success:
+ if verbose>0:
+ print("calculation of the temperature failed, calibration could not be applied")
+ return False,time,frames,False
+ else:
+
+ for i in range(len(frames)):
+ frames[i]=(frames[i]*FOV).astype(np.float32)
+ if verbose>0:
+ print(datetime.datetime.now(),"correcting bad pixels")
+ frames=IR_tools.correct_images(frames,badpixels)
+ if verbose>0:
+ print(datetime.datetime.now(),"checking temperaturerange")
+ valid=True
+ # for i in range(len(frames)):
+ # try:
+ # check_temperature_range(time[0],np.max(frames)-273.15,np.min(frames[:][np.nonzero(frames[i])])-273.15,portnr,t_exp,cfilter)
+ # except Exception as E:
+ # print(E)
+ # valid=False
+ # i=len(frames)
+ date=int(program.split(".")[0])
+ if date>20180712 and date<20180720:
+ prog=get_program_from_PID(program)
+ if prog[0]:
+ # t0time=prog[1]['trigger']['0'][0]
+ starttime=prog[1]['trigger']['1'][0]
+ toff=time[0]-starttime
+ if toff>10e6:
+ print("time offset detected, offset is "+str(toff)+" ns, correcting")
+ time[:]=time[:]-toff
+ if give_ERROR:
+ return exist,time,frames,valid,error_images
+ else:
+ return exist,time,frames,valid
+ else:
+ raise Exception("unknown Program")
def get_temp_from_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,check_range=True):
if threads==1 or not fastDL:
@@ -1415,7 +1677,7 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_v
out_q=multiprocessing.Queue()
for i in range(threads):
print("Start Thread ",i+1)
- p = multiprocessing.Process(target=convert_raw_to_temp_thread, args=(portnr,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,))
+ p = multiprocessing.Process(target=convert_raw_to_temp_thread, args=(larchivepath,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,))
jobs.append(p)
p.start()
resultdict = []
@@ -1603,14 +1865,14 @@ def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_erro
if verbose>0:
print(datetime.datetime.now(),"background treatment")
# if fullbackground:
-
+ #sbackground=np.zeros(np.shape(images[0]),dtype=np.float32)+background
for i in range(len(images)):
- images[i]=(images[i]-background).astype(np.float32)
+ images[i]=np.clip((images[i]-background),a_min=0,a_max=None).astype(np.uint16)
# else:
# for i in range(len(images)):
# images[i]=images[i]-background
# images=np.array(images.clip(min=0),dtype=np.uint16)
- images=[np.round(im.clip(min=0)).astype(np.uint16) for im in images]
+# images=[np.round(im.clip(min=0)).astype(np.uint16) for im in images]
if verbose>0:
print(datetime.datetime.now(),"applying LUT")
LUT=np.array([LUT[1],LUT[2]])
@@ -1635,6 +1897,7 @@ def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_erro
raise Warning(E)
return False,[0]
+
def apply_NUC(images,gain,offset,gain_error=0,offset_error=0,give_error=False):
"""
apply_NUC(images=numpy array(time,width,height),gain,offset =numpy array(width,height) )
@@ -2088,124 +2351,49 @@ def get_program_from_PID(instring):
end of the copied functions
@author: thomsen
"""
-#%%
+#%% dummy run of the script
if __name__=='__main__':
print("local function calling")
- port=10
-# prog="20171108.015"
- prog="20171109.021"
-# prog="20171025.030"
-# prog="20171012.035"
- prog="20171005.024"
- prog="20171109.056"
- prog="20171207.050"
- program="20180712.027"
- port=20
- prog="20180724.030"
- port=51
- program="20180809.010"
-# plt.figure()
-# for port in [10,11,20,21,30,31,40,41,51]:
-# framerate=get_camera_framerate_by_program(port,program)
-# if framerate[0]:
-# plt.plot(framerate[1],framerate[2],label="AEF"+str(port))
-# plt.legend()
-# plt.grid()
- port=10
- prog="20180816.016"
- prog="20180816.018"#[0,6]
- port=21
- prog="20180822.017"
- status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[6,7],emi=0.82,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
-# status,time,images=download_raw_images_by_program_via_png(port,prog,time_s=1,threads=4,verbose=0)
-
-# test=get_NUC_by_program(port,prog,8,version=0,verbose=5)
-# print(len(images))
-# prog=get_program_from_PID(prog)
+ port="AEK51"
+ prog="20180925.065"
+ status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,1],emi=0.82,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
+ if status:
+ plt.figure()
+ plt.imshow(images[-1],vmin=330,vmax=1000,cmap=exJet)
+ cb=plt.colorbar()
+ cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20)
+ cb.ax.tick_params(labelsize=20)
+# port=41
+# program="20180925.013"
+# prog=get_program_from_PID(program)
# if prog[0]:
-# t0time=prog[1]['trigger']['0'][0]
-# starttime=prog[1]['trigger']['1'][0]
-# import h5py as h5
-# szene=h5.File("X:\\E4 Diagnostics\\QIR\Calibrations\\scene_models\\AEF"+str(port)+"\\AEF"+str(port)+"_scene_model.h5")
-# szene=h5.File("\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\Software\\QI-RealTime\\1.1.0\\QRT-Analysis\\release\\Config\\spatial_calibration\\AEF"+str(port)+"_scene_model.h5",'r')
-# szene=h5.File("X:\\E4 Diagnostics\\QIR\Data\\For Holger\\AEF21_scene_model_1531872000000000000.h5",'r')
-# Xloc=np.asarray(szene['x'])
-# Yloc=np.asarray(szene['y'])
-# Zloc=np.asarray(szene['z'])
-# CAD=np.asarray(szene['CAD'])
-# timpo=0.87
-# fid=IR_tools.find_nearest((np.asarray(time)-time[0])/1e9,timpo)
-# plt.figure()#
-# plt.imshow(CAD,cmap='gray')
-# plt.imshow(images[fid-8]-images[fid-28],vmin=0,vmax=60,cmap=exJet_trans)
-# cb=plt.colorbar()
-# cb.set_label("temperature difference K", rotation=270,labelpad=26,fontsize=26)
-# cb.ax.tick_params(labelsize=26)
-# plt.figure()#
-# plt.imshow(images[fid-20]-images[fid-40],vmin=0,vmax=40,cmap=exJet)
-# cb=plt.colorbar()
-# cb.set_label("temperature difference K", rotation=270,labelpad=26,fontsize=26)
-# cb.ax.tick_params(labelsize=26)
-# plt.figure()# # plt.imshow(CAD,cmap='gray')
-# plt.imshow(images[fid-40],vmin=300,vmax=400,cmap=exJet)
-# cb=plt.colorbar()
-# cb.set_label("temperature K", rotation=270,labelpad=26,fontsize=26)
-# cb.ax.tick_params(labelsize=26)
-# proglist=["20180801.014","20180801.015","20180801.016","20180801.018","20180801.019"]
-# portlist=[20,21,51]
-# version=0
-# time_s=30
-# for program in proglist:
-# for port in portlist:
-# prog=AKF_1.get_program_from_PID(program)
-# if prog[0]:
-# t0time=prog[1]['trigger']['0'][0]
-# starttime=prog[1]['trigger']['1'][0]
-# try:
-# stoptime=prog[1]['trigger']['6'][0]
-# except:
-# stoptime=prog[1]['upto']
-# success=True
-# OP=get_OP_by_time(starttime)
-# Cam=portcamdict[OP]['AEF'+str(port)]
-# if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
-# if version==0:
-# version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
-# larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
-# elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
-# if version==0:
-# version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
-# larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
-# else:
-# raise Exception("Port number does not fit the known cameras")
-# if type(time_s)==list:
-# tstart=time_s[0]
-# tstop=time_s[1]
-# if tstop<tstart:
-# raise Exception("endtime before starttime")
-# stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
-# stdate=stdate.isoformat()
-# enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
-# enddate=enddate.isoformat()
-# else:
-# stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
-# stdate=stdate.isoformat()
-# t0date=datetime.datetime.utcfromtimestamp((t0time-100)/1e9)
-# t0date=t0date.isoformat()
-# if time_s==0:
-# enddate=datetime.datetime.utcfromtimestamp(stoptime+100/1e9)
-# enddate=enddate.isoformat()
-# else:
-# enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)
-# enddate=enddate.isoformat()
-# #"2017-11-15 08:00:00"
-# try:
-# times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
-# except:
-# times=[]
-# try:
-# timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),stdate.replace("T"," "))#
-# except:
-# timest0=[]
-# print(program,"AEF"+str(port),len(timest0),len(times))
\ No newline at end of file
+# t0=prog[1]['trigger']['0'][0]
+# t1=prog[1]['trigger']['1'][0]
+# t6=prog[1]['trigger']['6'][0]
+# for port in [10]:
+## bla=download_raw_images_by_times(port,t0,t1,verbose=5)
+# bla=get_temp_from_raw_by_program(port,program,time_s=[0,4],emi=0.82,T_version=1,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=True,verbose=5)
+# import h5py as h5
+# File=h5.File("AEF"+str(port)+"_"+program+"_temp_v1_b.h5",'w')
+# imags=np.asarray(bla[2]).swapaxes(0,2).swapaxes(0,1)
+# shapi=np.shape(imags)
+# dset=File.create_dataset('images',shape=(shapi[0],shapi[1],shapi[2]),dtype='uint16',chunks=(shapi[0],shapi[1],1))
+# imags=np.ascontiguousarray(imags)
+# dset.write_direct(imags)
+# File.create_dataset('timestamps', data=list(bla[1]), dtype='uint64')#,compression="gzip")
+# File.close()
+# prog="20171109.045"
+# for port in [10,11,20,21,30,31,40,41,50,51]:
+# points=valid_background_rectangle[port]
+# status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,0.1],emi=0.82,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
+# plt.figure()
+# plt.imshow(images[0],vmin=330,vmax=600)
+# plt.savefig("AEF"+str(port)+"_ori.png")
+# plt.close()
+# dummy=images[0]#[points[1]:points[3],points[0]:points[2]]#FOV*image#
+# dummy[points[1]:points[3],points[0]:points[2]]=1500
+# plt.figure()
+# plt.imshow(dummy,vmin=330,vmax=600)
+# plt.savefig("AEF"+str(port)+"_boxed.png")
+# plt.close()
\ No newline at end of file