From 555d3ed154fc1f864ac6d6f1df8131fc3cf56d73 Mon Sep 17 00:00:00 2001
From: Holger Niemann <holger.niemann@ipp.mpg.de>
Date: Thu, 4 Oct 2018 10:29:15 +0200
Subject: [PATCH] Update V2.9.0: change multiprocessing to multithreading,
performance improved, less archivecalls per download
---
IR_image_tools.py | 4 +-
downloadversionIRdata.py | 353 ++++++++++++++++++++++-----------------
2 files changed, 206 insertions(+), 151 deletions(-)
diff --git a/IR_image_tools.py b/IR_image_tools.py
index 539109c..b5069d2 100644
--- a/IR_image_tools.py
+++ b/IR_image_tools.py
@@ -153,7 +153,7 @@ def check_backgroundframe(backgroundframe,threshold=50):
valid=False
return valid,np.mean(dataset)
-def read_bad_pixels_from_file(port, shot_no=None, program=None):
+def read_bad_pixels_from_file(port, shot_no=None, program=None,time_ns=None):
'''Reads bad pixels stored in *.bpx file on E4 server.
Requires one of the optional arguments shot_no or program.
IN
@@ -169,6 +169,8 @@ def read_bad_pixels_from_file(port, shot_no=None, program=None):
OP = get_OP_by_time(shot_no=shot_no)
elif program is not None:
OP = get_OP_by_time(program_str=program)
+ elif time_ns is not None:
+ OP = get_OP_by_time(time_ns=time_ns)
else:
raise Exception('read_bad_pixels_from_file: ERROR! Need either shot no. or program string.')
diff --git a/downloadversionIRdata.py b/downloadversionIRdata.py
index 34998aa..8647ea8 100644
--- a/downloadversionIRdata.py
+++ b/downloadversionIRdata.py
@@ -3,7 +3,7 @@
Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018
-Version: 2.8.0
+Version: 2.9.0
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
@author: holn
"""
@@ -22,8 +22,8 @@ from PIL import Image
from io import BytesIO
from os.path import join
import matplotlib.pyplot as plt
-try:
- import multiprocessing
+try:
+ import threading
import archivedb as AKF_2
fastDL=True
except Exception as E:
@@ -705,16 +705,13 @@ if fastDL:
intervalls.append(int(i*intervalSize))
intervalls.append(lnt)
jobs = []
- out_q=multiprocessing.Queue()
+ resultdict = []
for i in range(threads):
if verbose>0:
print("Start Thread ",i+1)
- p = multiprocessing.Process(target=download_raw_images_png_by_times_thread, args=(larchivepath,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,))
- jobs.append(p)
- p.start()
- resultdict = []
- for i in range(threads):
- resultdict.append(out_q.get())
+ TH=download_images_thread(i,larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,version)
+ jobs.append(TH)
+ TH.start()
for p in jobs:
p.join()
if verbose>0:
@@ -723,8 +720,7 @@ if fastDL:
for ele in resultdict:
order.append(ele[0])
if len(np.where(np.asarray(ele[1])==False)[0])>0:
- success=False
-
+ success=False
times=[]#np.array(resultdict[order.index(0)][2])
images=[]#np.array(resultdict[order.index(0)][3])
for i in range(threads):
@@ -733,71 +729,110 @@ if fastDL:
resultdict[order.index(i)]=[]
del resultdict
return success,np.array(times),images
-# except Exception as E:
-# raise Exception(E)
-
+
+ class download_images_thread(threading.Thread):
+ def __init__(self, threadID, larchivepath,times,resultlist,version=0):
+ threading.Thread.__init__(self)
+ self.threadID = threadID
+ self.times = times
+ self.archivepath = larchivepath
+ self.resultlist = resultlist
+ self.version = version
+ def run(self):
+ images=[]
+ time=[]
+ successes=[]
+ for i in self.times:
+ imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
+ if imag[0]:
+ images.append(np.array(imag[1],dtype=np.uint16))
+ time.append(i)
+ successes.append(True)
+ else:
+ successes.append(False)
+ self.resultlist.append([self.threadID,successes,time,images])
- def download_raw_images_png_by_times_thread(larchivepath,times,out_q,threadnumber,version=0):
- images=[]
- time=[]
- successes=[]
- for i in times:
- imag=download_last_raw_image_by_time(larchivepath,i-10,i+10,version=version)
- if imag[0]:
- images.append(np.array(imag[1],dtype=np.uint16))
-# print(np.min(imag[1]),np.max(imag[1]))
- time.append(i)
- successes.append(True)
- else:
- successes.append(False)
- out_q.put([threadnumber,successes,time,images])
- # return success,np.array(time),np.array(images,dtype=np.uint16)
-
- def convert_raw_to_temp_thread(larchivepath,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
- images=[]
- time=[]
- successes=[]
- for i in times:
- imag=download_last_raw_image_by_time(larchivepath,i-10,i+10,version=version)
- if imag[0]:
- images.append(np.array(imag[1],dtype=np.uint16))
- time.append(i)
- successes.append(True)
+ class convert_raw_to_temp_thread(threading.Thread):
+ def __init__(self,larchivepath,times,resultlist,threadID,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
+ threading.Thread.__init__(self)
+ self.threadID = threadID
+ self.times = times
+ self.archivepath = larchivepath
+ self.resultlist = resultlist
+ self.version = version
+ self.gain=gain
+ self.offset=offset
+ self.background=background
+ self.LUT=LUT
+ self.refT=refT
+ self.gain_error=gain_error
+ self.offset_error=offset_error
+ self.give_ERROR=give_ERROR
+ self.FOV=FOV
+ self.badpixels=badpixels
+ def run(self):
+ images=[]
+ time=[]
+ successes=[]
+ for i in self.times:
+ imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
+ if imag[0]:
+ images.append(np.array(imag[1],dtype=np.uint16))
+ time.append(i)
+ successes.append(True)
+ else:
+ successes.append(False)
+ if self.give_ERROR:
+ sucess,images,error_images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR)
else:
- successes.append(False)
- if give_ERROR:
- sucess,images,error_images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
- else:
- success,images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
-
- for i in range(len(images)):
- images[i]=(images[i]*FOV).astype(np.float32)
- print(datetime.datetime.now(),"correcting bad pixels")
- images=IR_tools.correct_images(images,badpixels)
- out_q.put([threadnumber,successes,time,images])
+ success,images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR)
+
+ for i in range(len(images)):
+ images[i]=(images[i]*self.FOV).astype(np.float32)
+ print(datetime.datetime.now(),"correcting bad pixels")
+ images=IR_tools.correct_images(images,self.badpixels)
+ self.resultlist.append([self.threadID,successes,time,images])
- def NUC_raw_thread(larchivepath,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
- images=[]
- time=[]
- successes=[]
- for i in times:
- imag=download_last_raw_image_by_time(larchivepath,i-10,i+10,version=version)
- if imag[0]:
- images.append(np.array(imag[1],dtype=np.uint16))
- time.append(i)
- successes.append(True)
+ class NUC_raw_thread(threading.Thread):
+ def __init__ (self,larchivepath,times,resultlist,threadID,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
+ threading.Thread.__init__(self)
+ self.threadID = threadID
+ self.times = times
+ self.archivepath = larchivepath
+ self.resultlist = resultlist
+ self.version = version
+ self.gain=gain
+ self.offset=offset
+ self.background=background
+ self.LUT=LUT
+ self.refT=refT
+ self.gain_error=gain_error
+ self.offset_error=offset_error
+ self.give_ERROR=give_ERROR
+ self.FOV=FOV
+ self.badpixels=badpixels
+ def run(self):
+ images=[]
+ time=[]
+ successes=[]
+ for i in self.times:
+ imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
+ if imag[0]:
+ images.append(np.array(imag[1],dtype=np.uint16))
+ time.append(i)
+ successes.append(True)
+ else:
+ successes.append(False)
+ if self.give_ERROR:
+ images,error_images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
else:
- successes.append(False)
- if give_ERROR:
- images,error_images=apply_NUC(images,gain,offset,gain_error,offset_error,give_ERROR)
- else:
- images=apply_NUC(images,gain,offset,gain_error,offset_error,give_ERROR)
-
- for i in range(len(images)):
- images[i]=(images[i]*FOV).astype(np.float32)
- print(datetime.datetime.now(),"correcting bad pixels")
- images=IR_tools.correct_images(images,badpixels)
- out_q.put([threadnumber,successes,time,images])
+ images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
+
+ for i in range(len(images)):
+ images[i]=(images[i]*self.FOV).astype(np.float32)
+ print(datetime.datetime.now(),"correcting bad pixels")
+ images=IR_tools.correct_images(images,self.badpixels)
+ self.resultlist.append([self.threadID,successes,time,images])
def download_raw_FLIR_images_via_png(t1,t6,time_s=0,version=0,threads=1,verbose=0):
larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
@@ -940,16 +975,13 @@ def download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=0):
tim.append(times)
versionss.append(versions[setn])
jobs = []
- out_q=multiprocessing.Queue()
+ resultdict = []
for i in range(threads):
if verbose>0:
print("Start Thread ",i+1)
- p = multiprocessing.Process(target=download_raw_images_png_by_times_thread, args=(larchivepaths[i],tim[i],out_q,i,versionss[i],))
- jobs.append(p)
- p.start()
- resultdict = []
- for i in range(threads):
- resultdict.append(out_q.get())
+ TH=download_images_thread(i,larchivepaths[i],tim[i],resultdict,versionss[i])
+ jobs.append(TH)
+ TH.start()
for p in jobs:
p.join()
if verbose>0:
@@ -1481,10 +1513,10 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
prog=get_program_from_PID(program)
if prog[0]:
t0=prog[1]['trigger']['0'][0]
- starttime=prog[1]['trigger']['1'][0]
- stoptime=prog[1]['trigger']['6'][0]
+ t1=prog[1]['trigger']['1'][0]
+ t6=prog[1]['trigger']['6'][0]
success=True
- OP=get_OP_by_time(starttime)
+ OP=get_OP_by_time(t1)
if type(portnr) is str:
try:
@@ -1507,7 +1539,7 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
else:
if FLIR:
print("to be implemented")
- exist,time,frames,mode,tsets,versions=download_raw_FLIR_images_via_png(t1=starttime,t6=stoptime,time_s=time_s,version=version,threads=threads,verbose=verbose-1)
+ exist,time,frames,mode,tsets,versions=download_raw_FLIR_images_via_png(t1=t1,t6=t6,time_s=time_s,version=version,threads=threads,verbose=verbose-1)
if exist:
if mode==1:
Psets=IR_tools.check_dublicates(tsets)
@@ -1515,7 +1547,7 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
Temp_Co=[]
for PS in range(4):
if PS in Psets:
- parlog=read_restdb_old(archivepath+"W7X/QSR07_FLIR/AEK51_raw_PS"+str(PS)+"_PARLOG/V"+str(versions[PS])+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime+20))
+ parlog=read_restdb_old(archivepath+"W7X/QSR07_FLIR/AEK51_raw_PS"+str(PS)+"_PARLOG/V"+str(versions[PS])+"/_signal.json?from="+str(t1-10)+"&upto="+str(t6+20))
if parlog[0]:
pari=parlog[2][0]['meta-data']
c1=[float(pari['Coeff0_'+str(PS)].split("\n")[0]),float(pari['Coeff1_'+str(PS)].split("\n")[0])]
@@ -1552,16 +1584,33 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
print("FLIR data not found")
return False,[0],[0],[]
else:
- cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version,back_emissivity,verbose=verbose-1)
+ cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0,t1,t6,emi,T_version,version,back_emissivity,verbose=verbose-1)
if not cexist:
if verbose>0:
print("Unable to load the complete calibration data for",program," please confirm that this data was uploaded")
return False,[0],[0],False
else:
+
+ Cam=portcamdict[OP]['AEF'+str(portnr)]
+ if OP=="OP1.2a":
+ if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+ if version==0:
+ version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(portnr)+"_raw_DATASTREAM")
+ larchivepath=archivepath+"Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+ elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
+ if version==0:
+ version=get_latest_version("QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM")
+ larchivepath=archivepath+"W7X/"+"QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+ else:
+ raise Exception("Port number does not fit the known cameras")
+ elif OP=="OP1.2b":
+ if version==0:
+ version=get_latest_version("QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM")
+ larchivepath=archivepath+"W7X/"+"QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
if verbose>0:
print(datetime.datetime.now(),"raw download start")
if fastDL:
- exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_s,version,threads,verbose=verbose-1)
+ exist,time,frames=download_raw_images_by_time_via_png(larchivepath,starttime=t1,stoptime=t6,time_s=time_s,version=version,threads=threads,verbose=verbose-1)
else:
exist,time,frames=download_raw_images_by_program(portnr,program,time_s,version,verbose=verbose-1)
if exist:
@@ -1578,6 +1627,7 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
if use_firstframe_as_background:
bim=apply_NUC([frames[0]],gain,offset)
background=get_average_background_recangle(portnr,bim[0])
+
if give_ERROR:
success,frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
else:
@@ -1606,10 +1656,10 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
# i=len(frames)
date=int(program.split(".")[0])
if date>20180712 and date<20180720:
- prog=get_program_from_PID(program)
+# prog=get_program_from_PID(program)
if prog[0]:
# t0time=prog[1]['trigger']['0'][0]
- starttime=prog[1]['trigger']['1'][0]
+ starttime=t1#prog[1]['trigger']['1'][0]
toff=time[0]-starttime
if toff>10e6:
print("time offset detected, offset is "+str(toff)+" ns, correcting")
@@ -1637,11 +1687,11 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_v
if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
if version==0:
version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(portnr)+"_raw_DATASTREAM")
- larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+ larchivepath=archivepath+"W7X/"+"QRT_INFRATEC/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
if version==0:
version=get_latest_version("QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM")
- larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+ larchivepath=archivepath+"W7X/"+"QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
else:
raise Exception("Port number does not fit the known cameras")
if type(time_s)==list:
@@ -1674,15 +1724,13 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_v
intervalls.append(int(i*intervalSize))
intervalls.append(lnt)
jobs = []
- out_q=multiprocessing.Queue()
+ resultdict = []
for i in range(threads):
print("Start Thread ",i+1)
- p = multiprocessing.Process(target=convert_raw_to_temp_thread, args=(larchivepath,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,))
+# p = multiprocessing.Process(target=convert_raw_to_temp_thread, args=(larchivepath,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,))
+ p=convert_raw_to_temp_thread(larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels)
jobs.append(p)
- p.start()
- resultdict = []
- for i in range(threads):
- resultdict.append(out_q.get())
+ p.start()
for p in jobs:
p.join()
print("all threads are done")
@@ -1751,27 +1799,29 @@ def get_nuced_raw_by_program(portnr,program,time_s=0,version=0,threads=1,give_ER
else:
return exist,time,frames,valid
-def get_nuced_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,check_range=True):
- if threads==1 or not fastDL:
- return get_nuced_raw_by_program(portnr,program,time_s,emi,T_version,version,threads,give_ERROR)
- else:
- background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version)
- FOV=get_FOV_mask(portnr)
- prog=get_program_from_PID(program)
- if prog[0]:
- starttime=prog[1]['trigger']['1'][0]
- stoptime=prog[1]['trigger']['6'][0]
+def get_nuced_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_version=2,version=1,threads=1,give_ERROR=False,check_range=True):
+ prog=get_program_from_PID(program)
+ if prog[0]:
+ t0=prog[1]['trigger']['0'][0]
+ t1=prog[1]['trigger']['1'][0]
+ t6=prog[1]['trigger']['6'][0]
+ if threads==1 or not fastDL:
+ return get_nuced_raw_by_program(portnr,program,time_s,emi,T_version,version,threads,give_ERROR)
+ else:
+ exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0,t1,t6,emi,T_version,version)
+ FOV=get_FOV_mask(portnr)
+
success=True
- OP=get_OP_by_time(starttime)
+ OP=get_OP_by_time(t1)
Cam=portcamdict[OP]['AEF'+str(portnr)]
if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
if version==0:
version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(portnr)+"_raw_DATASTREAM")
- larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+ larchivepath=archivepath+"W7X/"+"QRT_INFRATEC/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
if version==0:
version=get_latest_version("QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM")
- larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
+ larchivepath=archivepath+"W7X/"+"QRT_IRCAM/"+"AEF"+str(portnr)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
else:
raise Exception("Port number does not fit the known cameras")
if type(time_s)==list:
@@ -1779,20 +1829,19 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_versi
tstop=time_s[1]
if tstop<tstart:
raise Exception("endtime before starttime")
- stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
+ stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
stdate=stdate.isoformat()
- enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
+ enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)
enddate=enddate.isoformat()
else:
- stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
+ stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
stdate=stdate.isoformat()
if time_s==0:
- enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)
+ enddate=datetime.datetime.utcfromtimestamp(t6/1e9)
enddate=enddate.isoformat()
else:
- enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)
+ enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_s)
enddate=enddate.isoformat()
-
times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
lnt=len(times)
tim=[]
@@ -1804,15 +1853,13 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_versi
intervalls.append(int(i*intervalSize))
intervalls.append(lnt)
jobs = []
- out_q=multiprocessing.Queue()
+ resultdict = []
for i in range(threads):
print("Start Thread ",i+1)
- p = multiprocessing.Process(target=NUC_raw_thread, args=(portnr,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,))
+# p = multiprocessing.Process(target=NUC_raw_thread, args=(portnr,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,))
+ p = NUC_raw_thread(larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels)
jobs.append(p)
- p.start()
- resultdict = []
- for i in range(threads):
- resultdict.append(out_q.get())
+ p.start()
for p in jobs:
p.join()
print("all threads are done")
@@ -1820,7 +1867,7 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_versi
for ele in resultdict:
order.append(ele[0])
if len(np.where(np.asarray(ele[1])==False)[0])>0:
- success=False
+ success=False
times=[]#np.array(resultdict[order.index(0)][2])
images=[]#np.array(resultdict[order.index(0)][3])
for i in range(threads):
@@ -1828,15 +1875,16 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_versi
times=times+resultdict[order.index(i)][2]
del resultdict
valid=True
- else:
- success=False
- times=[0]
- images=[0]
- valid=False
- error_images=[0]
+# else:
+# success=False
+# times=[0]
+# images=[0]
+# valid=False
+# error_images=[0]
+ print(larchivepath)
if give_ERROR:
- return success,np.array(times),images,valid,error_images
+ return success,np.array(times),images,valid#,error_images
else:
return success,np.array(times),images,valid
@@ -1950,30 +1998,32 @@ def get_cooling_water_temperature_by_time(time_ns):
result_out=[False,0,0]
return result_in[0] and result_out[0],inlet,outlet
-def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivity=0.82, verbose=0):
+def get_calib_data(port,t0,t1,t6,emissivity=0.8,Temp_V=2,version=0,back_emissivity=0.82, verbose=0):
"""
get_calib_data
return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
"""
- prog=get_program_from_PID(program)
- OP=IR_tools.get_OP_by_time(program_str=program)
+# prog=get_program_from_PID(program)
+ OP=IR_tools.get_OP_by_time(time_ns=t0)#program_str=program)
exist=True
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
- if prog[0]:
- t0=prog[1]['trigger']['0'][0]
- t1=prog[1]['trigger']['1'][0]
- t6=prog[1]['trigger']['6'][0]
+# if prog[0]:
+# t0=prog[1]['trigger']['0'][0]
+# t1=prog[1]['trigger']['1'][0]
+# t6=prog[1]['trigger']['6'][0]
# except Exception as E:
# print("Warning: Program was not found: ",E)
# return 0,0,0,0,0,0,0,0,0,0
- else:
- raise Warning("cannot find the program!")
- return False,0,0,0,0,0,0,0,0,0,0
+# else:
+# raise Warning("cannot find the program!")
+# return False,0,0,0,0,0,0,0,0,0,0
+ print(OP,port)
Camera=portcamdict[OP]['AEF'+str(port)]
if Camera.split("_")[0]=="Infratec":
- cf=get_INFRATEC_filter_by_program(program,version)
+# cf=get_INFRATEC_filter_by_program(program,version)
+ cf=get_INFRATEC_filter_by_times(starttime=t1,stoptime=t6)
if cf[0]:
cfilter=cf[2]
else:
@@ -2096,7 +2146,7 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
if verbose>0:
print(datetime.datetime.now(),"Scanning for bad pixel")
# initial list from config files
- init_bp_list = IR_tools.read_bad_pixels_from_file(port, program=program)
+ init_bp_list = IR_tools.read_bad_pixels_from_file(port, time_ns=t1)
# find more bad pixel
badpixels = find_badpixels(port, gain, offset, init_bp_list=init_bp_list, niterations=10, tolerance=10, verbose=verbose-1)
return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
@@ -2355,15 +2405,18 @@ end of the copied functions
if __name__=='__main__':
print("local function calling")
- port="AEK51"
+ port="AEF51"
prog="20180925.065"
- status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,1],emi=0.82,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
- if status:
- plt.figure()
- plt.imshow(images[-1],vmin=330,vmax=1000,cmap=exJet)
- cb=plt.colorbar()
- cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20)
- cb.ax.tick_params(labelsize=20)
+# status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,4],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
+# if status:
+# plt.figure()
+# plt.imshow(images[-1],vmin=330,vmax=1000,cmap=exJet)
+# cb=plt.colorbar()
+# cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20)
+# cb.ax.tick_params(labelsize=20)
+
+# bla=get_temp_from_raw_by_program_fullthreads(51,prog,time_s=[0,4],threads=4)
+ bla2=get_nuced_raw_by_program_fullthreads(51,prog,time_s=[0,4],threads=4)
# port=41
# program="20180925.013"
# prog=get_program_from_PID(program)
--
GitLab