diff --git a/IR_config_constants.py b/IR_config_constants.py
index d92c359622de28f647bdebaf62f85670f205e252..7c880c38dd8b2f1d87815cf7c97be6c1ca95ef5b 100644
--- a/IR_config_constants.py
+++ b/IR_config_constants.py
@@ -11,7 +11,11 @@ IRCamColdframes_fittingpath=calibpath+"IRCamColdframes_fitting\\"
#IRCamRefImagespath = calibpath+'IRCamReferenceImages\\'
IRCAMBadPixels_path = calibpath + 'IRCAMBadPixels\\'
heatflux_requestlist_path="\\\\x-drive\\Diagnostic-logbooks\\QRT-DivertorThermography\\ArchiveDB_heatflux_pipeline\\"
-parameter_file_path = ".\\data"
+# get local parameter directory from script location
+import os.path as ospath
+parameter_file_path = ospath.join(ospath.dirname(__file__), 'data')
+
+
try:
# import sys
path=str(repr(__file__)).split("IR_config_constants")[0].split("'")[1]+"upload_config"
@@ -28,6 +32,7 @@ except Exception as E:
archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
print("config loading failed",E)
# raise Exception("Config file was not loaded properly",E)
+testarchivepath = "http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
#%% Dictionaries
project="W7X"
diff --git a/IR_image_tools.py b/IR_image_tools.py
index fe61edd1ddd01979a01bd69fc97be282e16dddd4..8622cb3d87b143dee5b478614d26970366589f93 100644
--- a/IR_image_tools.py
+++ b/IR_image_tools.py
@@ -21,9 +21,6 @@ import datetime
#import h5py
#import glob
-# set working directory to local directory of script
-os.chdir(os.path.dirname(__file__))
-
def get_OP_by_time(time_ns=None, shot_no=None, program_str=None):
'''Derives operation phase (OP) of W7-X based on either:
a nanosacond time stamp, a MDSplus style shot no. or a program ID.
@@ -57,7 +54,7 @@ def get_OP_by_time(time_ns=None, shot_no=None, program_str=None):
return "OP1.2a"
else:
return None
- elif dateOP.year == 2018:
+ elif dateOP.year >= 2018:
return "OP1.2b"
elif dateOP.year <= 2016 and dateOP.year >= 2015:
if (dateOP.year == 2016 and dateOP.month <= 3) or (dateOP.year == 2015 and dateOP.month == 12):
diff --git a/downloadversionIRdata.py b/downloadversionIRdata.py
index a0fbbf2538c4a74e401727a649e1058aa1267cea..ad4b73d36b5b76fd5390db8d767bcfda0376e7b2 100644
--- a/downloadversionIRdata.py
+++ b/downloadversionIRdata.py
@@ -11,7 +11,7 @@ Version: 3.2.5
import numpy as np
import IR_image_tools as IR_tools
-from IR_config_constants import archivepath,portcamdict,camlimdict, \
+from IR_config_constants import archivepath,testarchivepath,portcamdict,camlimdict, \
valid_FOV_circle,valid_background_rectangle, \
TC_port,TC_channel,IRCamColdframes_fittingpath, \
exJet,portpathdict,project,project_ana, \
@@ -30,6 +30,9 @@ import matplotlib.pyplot as plt
try:
import threading
import archivedb as AKF_2
+ # disbale deprecation warnings, because of timezone warning spam
+ import warnings
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
fastDL=True
except Exception as E:
print(E)
@@ -44,27 +47,29 @@ except:
config_path=""
-def get_latest_version(stream,project="W7X",Test=False,t_from=None,t_to=None,program=None):
+def get_latest_version(stream,project="W7X",testmode=False,t_from=None,t_to=None,
+ program=None,verbose=0):
"""Find out latest version of given stream
:param stream url stream of interest
:return int of version number of None if non-versioned or non-existing stream
author: G. Schlisio, edit by holn
"""
- if Test:
- base="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
+ if testmode:
+ base=testarchivepath
else:
base=archivepath
-
if program==None and t_from==None and t_to==None:
request =urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})
else:
if program is not None:
- prog=get_program_from_PID(program)
- if prog[0]:
- t_from=prog[1]['trigger']['0'][0]
- t_to=prog[1]['trigger']['6'][0]
+# prog=get_program_from_PID(program)
+# if prog[0]:
+# t_from=prog[1]['trigger']['0'][0]
+# t_to=prog[1]['trigger']['6'][0]
+ exist, t_from, _, t_to = get_trigger_from_PID(program, stream, testmode, verbose=verbose-1)
+
elif t_from is not None and t_to==None:
t_to=int(t_from+100e9)
elif t_from==None and t_to is not None:
@@ -129,7 +134,8 @@ def read_program(timestamp_start,timestamp_end=0,tol=60):
-def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
+def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0,
+ testmode=False,verbose=0):
"""
download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
time in ns
@@ -140,7 +146,10 @@ def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, ve
port = 11
elif port == 11:
port = 21
- larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
@@ -149,7 +158,7 @@ def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, ve
print("download_LUT: Error! Camera unknown, stopping here.")
raise Exception
if version==0:
- version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM",t_from=time)
+ version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM",t_from=time,testmode=testmode)
if verbose>0:
print("download_LUT: LUT V"+str(version)+" is used")
#time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
@@ -194,27 +203,35 @@ def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_s
print("read_LUT_from_file: ERROR in loading V3 LUTs",E)
return False, []
-def download_NUC_by_program(port,program,exposure,version=0):
- prog=get_program_from_PID(program)
+def download_NUC_by_program(port,program,exposure,version=0,testmode=False,verbose=0):
+ '''
+ '''
+# prog=get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
- if prog[0]:
- starttime=prog[1]['trigger']['0'][0]
- stoptime=prog[1]['trigger']['1'][0]
- return download_NUC_by_times(port,starttime,stoptime,exposure,version)
+# if prog[0]:
+# starttime=prog[1]['trigger']['0'][0]
+# stoptime=prog[1]['trigger']['1'][0]
+ exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
+ return download_NUC_by_times(port,starttime,stoptime,exposure,version,verbose=verbose)
else:
# except:
print("download_NUC_by_program: cannot find the program")
return False,0,0
-def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
- project="W7X"
+def download_NUC_by_times(port,starttime,stoptime,exposure,testmode=False,version=0,verbose=0):
+ '''
+ '''
OP=IR_tools.get_OP_by_time(time_ns=starttime)
- larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
# NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if version==0:
- version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM",t_from=starttime)
+ version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM",t_from=starttime,testmode=testmode)
try:
res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
signal_list = json.loads(res.read().decode('utf-8'))
@@ -249,29 +266,36 @@ def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
else:
return False,0,0
-def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
- prog=get_program_from_PID(program)
+def get_NUC_by_program(port,program,exposure,version=0,testmode=False,verbose=0):
+ '''
+ '''
+# prog=get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
- if prog[0]:
- starttime=prog[1]['trigger']['0'][0]
- stoptime=prog[1]['trigger']['1'][0]
- return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose)
+# if prog[0]:
+# starttime=prog[1]['trigger']['0'][0]
+# stoptime=prog[1]['trigger']['1'][0]
+ exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
+ return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,testmode=testmode,verbose=verbose)
else:
# except:
- print("get_NUC_by_program: Error! cannot find the program")
- return False,0,0
+ print("get_NUC_by_program: Warning! cannot find the trigger timestamps for program", program)
+# return False,0,0
+ OP=IR_tools.get_OP_by_time(program_str=program)
+ return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,testmode=testmode,OP=OP,verbose=verbose)
-def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
+def get_NUC_by_times(port,starttime,stoptime,t_exp,version=0,testmode=False,OP=None,verbose=0):
"""
Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera
in a given port and time interval. Depending on time and camera, reconstruct
- teh cold frame and/or compute the gain,offset from the hot and cold frame.
+ the cold frame and/or compute the gain,offset from the hot and cold frame.
"""
- OP=IR_tools.get_OP_by_time(time_ns=endtime)
+ if OP is None:
+ OP=IR_tools.get_OP_by_time(time_ns=stoptime)
if OP=="OP1.2a":
- t1=endtime
+ t1=stoptime
t0=starttime
prog=read_program(t1)
program=prog[1]['id']
@@ -279,15 +303,15 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
if verbose>0:
print("get_NUC_by_times: rebuilding coldframe")
#use any way the rebuilt coldframe.
- sT = get_sensor_temp_by_program(port, program)[2][0]
+ sT = get_sensor_temp_by_program(port, program,testmode=testmode)[2][0]
# coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
- gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
+ gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp,testmode=False)
if gotit:
coldref=hotcold[1]
hotref=hotcold[0]
else:
raise Exception("get_NUC_by_times: unable to download reference frames")
- filestring = 'AEF' + str(port) + '_et' + str(t_exp)
+ filestring = 'AEF' + str(port) + '_et' + str(int(t_exp))
amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
@@ -304,13 +328,13 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
if verbose>0:
print('get_NUC_by_times: downloading NUC')
# coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
- gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
+ gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp,testmode=False)
if gotit:
coldref=hotcold[1]
hotref=hotcold[0]
else:
raise Exception("get_NUC_by_times: unable to download reference frames")
- NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
+ NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version,testmode)
if NUC_DL[0]==False:
raise Exception("get_NUC_by_times: NUC was not found")
else:
@@ -325,38 +349,43 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
else:
if verbose>0:
print('get_NUC_by_times: downloading NUC')
- return download_NUC_by_times(port,starttime,endtime,t_exp)
+ return download_NUC_by_times(port,starttime,stoptime,t_exp,testmode=testmode)
elif OP=="OP1.2b":
gain_error=0
offset_error=0
- gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
+ gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp,testmode=False)
if gotit:
coldref=hotcold[1]
hotref=hotcold[0]
else:
raise Exception("get_NUC_by_times: unable to download reference frames")
### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
- exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0,verbose=verbose-1)
+ try:
+ exist,_,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,stoptime,frametype=0,version=0,testmode=testmode,verbose=verbose-1)
+ except:
+ exist = False
if exist:
cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
for ele in coldframes:
cold+=ele
cold=np.asarray(cold/len(coldframes),dtype=np.uint16)
else:
- return False,[0],[0]
+ print('get_NUC_by_times: Warning! no cold frames found. will use reference cold frame...')
+ cold = np.copy(coldref)
+# return False,[0],[0]
gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)
return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
else:
raise Exception("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP")
-def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0,verbose=0):
+def download_calibration_raw_files_by_time(port,t_exp,starttime,stoptime,frametype=0,version=0,testmode=False,verbose=0):
"""
- download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
+ download_calibration_raw_files_by_time(port,t_exp,starttime,stoptime,frametype=0,version=0)
frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
"""
- gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(endtime-20e6))
- OP=IR_tools.get_OP_by_time(time_ns=endtime)
+ gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(stoptime-20e6),testmode=testmode)
+ OP=IR_tools.get_OP_by_time(time_ns=stoptime)
if gotit:
expinds=np.where(texp_t==t_exp)[0]
if len(expinds)==0:
@@ -365,7 +394,7 @@ def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametyp
else:
print("download_calibration_raw_files_by_time: Error! exposure time not found")
return False,[0],[0]
- gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(endtime-20e6))
+ gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(stoptime-20e6),testmode=testmode)
if gotitf:
typiinds=np.where(values_f[expinds]==frametype)[0]
if len(typiinds)>0:
@@ -380,11 +409,11 @@ def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametyp
turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
if frametype==0:
ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
- print((ref_t[1]-ref_t[0])/1e9)
+# print((ref_t[1]-ref_t[0])/1e9)
elif frametype==1:
# print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))
ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
- print((ref_t[1]-ref_t[0])/1e9)
+# print((ref_t[1]-ref_t[0])/1e9)
else:
raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
else:
@@ -402,13 +431,16 @@ def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametyp
else:
raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
# return False,[0],[0]
- t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
+ t1date=datetime.datetime.utcfromtimestamp((stoptime-100)/1e9)
t1date=t1date.isoformat()
t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
t0date=t0date.isoformat()
if version==0:
- version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=endtime)
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
+ version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime,testmode=testmode)
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
t_offset=timest0[-1][0]-time_t[0]
if t_offset>10e6:
@@ -417,15 +449,19 @@ def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametyp
else:
t_offset=0
# print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
- return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset,verbose=verbose-1)
+ return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset,testmode=testmode,verbose=verbose-1)
-def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
+def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,
+ testmode=False,version=0):
"""
Loads the most recent hot and cold calibration frames for a starttime.
Uses first calibration frames if time is not defined.
"""
OP=IR_tools.get_OP_by_time(time_ns=starttime)
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
# NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if version==0:
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime)
@@ -470,29 +506,35 @@ def download_hot_cold_reference_by_times(port,exposure,starttime=150390720000000
else:
return False,0,0
-def download_background_by_program(port,program,exposure,version=0):
+def download_background_by_program(port,program,exposure,version=0,testmode=False,verbose=0):
'''
returned exist,time,frame
'''
- prog=get_program_from_PID(program)
+# prog=get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
- if prog[0]:
- starttime=prog[1]['trigger']['0'][0]
- stoptime=prog[1]['trigger']['1'][0]
+# if prog[0]:
+# starttime=prog[1]['trigger']['0'][0]
+# stoptime=prog[1]['trigger']['1'][0]
+ exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
return download_background_by_times(port,starttime,stoptime,exposure,version)
else:#except:
print("download_background_by_program: Error! cannot find the program {0}".format(program))
return False,0,0
-def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0,verbose=0):
+def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,
+ version=0,testmode=False,verbose=0):
OP=IR_tools.get_OP_by_time(time_ns=starttime)
if OP=="OP1.2a":
stream=portpathdict[OP]["AEF"+str(port)]+"background_"
- larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
if version==0:
- version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
+ version=get_latest_version(stream+"DATASTREAM",t_from=starttime,testmode=testmode)
try:
res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
signal_list = json.loads(res.read().decode('utf-8'))
@@ -528,7 +570,7 @@ def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=
else:
return False,0,0
elif OP=="OP1.2b":
- NUC_DL=get_NUC_by_times(port=port,starttime=starttime,endtime=stoptime,t_exp=exposure,version=version,verbose=verbose-1)#download_NUC_by_times(port,t0,t1,t_exp,version)
+ NUC_DL=get_NUC_by_times(port,starttime,stoptime,t_exp=exposure,version=version,testmode=testmode,verbose=verbose-1)#download_NUC_by_times(port,t0,t1,t_exp,version)
if NUC_DL[0]==False:
raise Warning("download_background_by_times: NUC was not found")
return False,0,0
@@ -544,11 +586,11 @@ def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=
offset[offset<-1e100]=0
return get_NUCed_background_by_times(port=port,t0=starttime,t1=stoptime,t_exp=exposure,cfilter=camera_filter,gain=gain,offset=offset,version=version,verbose=verbose-1)
-def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False,verbose=0):
+def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,testmode=False,plot_it=False,verbose=0):
"""
OP1.2b function
"""
- exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version,verbose=verbose-1)
+ exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version,testmode=testmode,verbose=verbose-1)
camera=portcamdict["OP1.2b"]["AEF"+str(port)]
if verbose>50:
print("camera is",camera)
@@ -569,7 +611,8 @@ def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0
return exist,btime[0],background
-def get_NUCed_coldframe_by_program(port, program, exposure=None, version=0, plot_it=False, verbose=0):
+def get_NUCed_coldframe_by_program(port, program, exposure=None, version=0,
+ testmode=False, plot_it=False, verbose=0):
"""
Load the raw cold refences frame taken bofore this program and NUC it with
the NUC of the previous program (or next one, if the previous does not exist).
@@ -603,18 +646,18 @@ def get_NUCed_coldframe_by_program(port, program, exposure=None, version=0, plot
"""
if exposure is None:
try:
- exist, t, exposure = get_exposure_by_program(port, program, version=version)
+ exist, t, exposure = get_exposure_by_program(port, program, version=version, testmode=testmode)
except:
print('get_NUCed_coldframe_by_program: ERROR! no exposure time found!')
return False, []
t_exp = exposure[0]
else:
- if verbose>0:
- print('get_NUCed_coldframe_by_program: using exposure time {0}'.format(exposure))
t_exp = exposure
+ if verbose>0:
+ print('get_NUCed_coldframe_by_program: using exposure time {0}'.format(t_exp))
try:
- exist, data, desc = get_NUC_by_program(port, program, t_exp, version=version, verbose=verbose-1)
+ exist, data, desc = get_NUC_by_program(port, program, t_exp, version=version, testmode=testmode, verbose=verbose-1)
except:
print('get_NUCed_coldframe_by_program: ERROR! no cold frame found!')
return False, []
@@ -624,33 +667,27 @@ def get_NUCed_coldframe_by_program(port, program, exposure=None, version=0, plot
porg_nr = int(program.split('.')[1])
ref_program = '{0}.{1:03d}'.format(program.split('.')[0], porg_nr-1)
- exist2, data, desc = get_NUC_by_program(port, ref_program, t_exp, version=version, verbose=verbose-1)
+ exist2, data, desc = get_NUC_by_program(port, ref_program, t_exp, version=version, testmode=testmode, verbose=verbose-1)
if exist2:
coldframe = apply_NUC([cold_raw],data[0],data[1])[0]
if verbose>0:
print('get_NUCed_coldframe_by_program: using NUC of program {0}'.format(ref_program))
else:
- ref_program = '{0}.{1:03d}'.format(program.split('.')[0], porg_nr+1)
- if verbose>0:
- print('get_NUCed_coldframe_by_program: program {0} has no predecessor.\n --> using NUC of {1}'.format(program, ref_program))
- exist2, data, desc = get_NUC_by_program(port, ref_program, t_exp, version=version, verbose=verbose-1)
- if exist2:
- coldframe = apply_NUC([cold_raw],data[0],data[1])[0]
- else:
- print('get_NUCed_coldframe_by_program: WARNING! no adjacent program found! --> cold frame is not NUCed')
- coldframe = cold_raw
+ print('get_NUCed_coldframe_by_program: WARNING! no adjacent program found! --> cold frame is not NUCed')
+ coldframe = cold_raw
if plot_it:
plt.figure(figsize=[8,4])
plt.subplot(1,2,1)
plt.imshow(cold_raw, vmin=np.percentile(cold_raw, 1), vmax=np.percentile(cold_raw, 99) )
plt.colorbar()
- plt.title('raw cold frame\nof {0}'.format(program))
+ plt.title('averaged raw\nof {0}'.format(program))
plt.subplot(1,2,2)
plt.imshow(coldframe, vmin=np.percentile(coldframe, 1), vmax=np.percentile(coldframe, 99) )
plt.colorbar()
plt.title('NUCed based on\ngain/offset from {0}'.format(ref_program))
- plt.tight_layout()
+ plt.suptitle('cold frame of port {0}'.format(port))
+ plt.tight_layout(rect=[0, 0, 1, 0.95])
plt.show()
else:
print('get_NUCed_coldframe_by_program: ERROR! No cold frame found for program {0}!'.foramt(program))
@@ -659,40 +696,45 @@ def get_NUCed_coldframe_by_program(port, program, exposure=None, version=0, plot
return exist, coldframe
-def download_raw_images_by_program(port,program,time_window=0,version=0,verbose=0):
+def download_raw_images_by_program(port,program,time_window=0,version=0,testmode=False,verbose=0):
"""
"""
+ exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
# prog=AKF_1.get_program_from_PID(program)
- try:
- t_program = AKF_2.get_program_from_to(program)
- prog =AKF_2.get_program_list(t_program[0], t_program[1])
-# if prog[0]:
- starttime=prog[0]['trigger']['1'][0]
- stoptime=prog[0]['trigger']['6'][0]
+# try:
+# t_program = AKF_2.get_program_from_to(program)
+# prog =AKF_2.get_program_list(t_program[0], t_program[1])
+## if prog[0]:
+# starttime=prog[0]['trigger']['1'][0]
+# stoptime=prog[0]['trigger']['6'][0]
+ if exist:
if type(time_window)==list or type(time_window)==np.ndarray:
if len(time_window)>2 or time_window[0]>1000:
raise Exception("download_raw_images_by_program: invalid input for the time_window!")
tstart=time_window[0]
tstop=time_window[1]
if tstop<tstart:
- raise Exception("download_raw_images_by_program: endtime before starttime")
- return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,verbose=verbose)
+ raise Exception("download_raw_images_by_program: t_stop before t_start")
+ return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,testmode=testmode,verbose=verbose)
else:
if time_window==0:
- return download_raw_images_by_times(port,starttime,stoptime,version,verbose=verbose)
+ return download_raw_images_by_times(port,starttime,stoptime,version,testmode=testmode,verbose=verbose)
else:
- return download_raw_images_by_times(port,starttime,int(starttime+time_window*1e9),version,verbose=verbose)
- except:
+ return download_raw_images_by_times(port,starttime,int(starttime+time_window*1e9),version,testmode=testmode,verbose=verbose)
+ else:
print("download_raw_images_by_program: Error! cannot find the program")
return False,0,0
-def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
+def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,testmode=False,verbose=0):
"""
"""
OP=IR_tools.get_OP_by_time(time_ns=starttime)
if version==0:
- version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime)
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
+ version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime,testmode=testmode)
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
return download_images_by_times(larchivepath,starttime=starttime,stoptime=stoptime,version=version,intervalSize=intervalSize,verbose=verbose)
def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
@@ -758,27 +800,38 @@ def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalS
return False, 0,-1
if fastDL:
- def download_raw_images_by_program_via_png(port,program,time_window=0,version=0,threads=1,verbose=0):
+ def download_raw_images_by_program_via_png(port,program,time_window=0,version=0,
+ threads=1,testmode=False,verbose=0):
"""
"""
- prog=get_program_from_PID(program)
-# try:
-# t_program = AKF_2.get_program_from_to(program)
-# prog =AKF_2.get_program_list(t_program[0], t_program[1])
- if not prog[0]:
- print("download_raw_images_by_program_via_png: Error! program not found, cannot download the data")
- return False,[0],[0]
+# prog=get_program_from_PID(program)
+## try:
+## t_program = AKF_2.get_program_from_to(program)
+## prog =AKF_2.get_program_list(t_program[0], t_program[1])
+# if not prog[0]:
+# print("download_raw_images_by_program_via_png: Error! program not found, cannot download the data")
+# return False,[0],[0]
+# starttime=prog[1]['trigger']['1'][0]
+# stoptime=prog[1]['trigger']['6'][0]
+
+ trigger_exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if not trigger_exist:
+ print("download_raw_images_by_program_via_png: Error! trigger not found, cannot download the data")
+ OP=IR_tools.get_OP_by_time(time_ns=starttime)
+
+ # get raw data from latest or requested version
+ if version==0:
+ version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",testmode=testmode,program=program)
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
else:
- starttime=prog[1]['trigger']['1'][0]
- stoptime=prog[1]['trigger']['6'][0]
- OP=IR_tools.get_OP_by_time(time_ns=starttime)
- if version==0:
- version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime)
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
- return download_images_by_time_via_png(larchivepath,starttime,stoptime,
- time_window=time_window,threads=threads,
- version=version,verbose=verbose-1)
-
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
+
+ return download_images_by_time_via_png(larchivepath,starttime,stoptime,
+ time_window=time_window,threads=threads,
+ version=version,verbose=verbose-1)
+
+
def download_images_by_time_via_png(larchivepath,starttime,stoptime,time_window=0,
threads=1,version=0,verbose=0,framerate='max'):
"""
@@ -788,25 +841,37 @@ if fastDL:
tstart=time_window[0]
tstop=time_window[1]
if tstop<tstart:
- raise Exception("download_images_by_time_via_png: endtime before starttime")
- stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
- stdate=stdate.isoformat()
- enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
- enddate=enddate.isoformat()
+ raise Exception("download_images_by_time_via_png: t_stop before t_start")
+ time_from = (starttime-100)+int(tstart*1E9)
+ if stoptime<time_from:
+ raise Exception("download_images_by_time_via_png: time_window after stoptime")
+ time_to = starttime+int(tstop*1E9)
+ if stoptime<time_to:
+ time_to = stoptime
+ print('download_images_by_time_via_png: Warning! time_window cropped by end of data')
+# stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
+# stdate=stdate.isoformat()
+# enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
+# enddate=enddate.isoformat()
else:
- stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
- stdate=stdate.isoformat()
+# stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
+# stdate=stdate.isoformat()
+ time_from = starttime-100
if time_window==0:
- enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)
- enddate=enddate.isoformat()
+ time_to = stoptime
+# enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)
+# enddate=enddate.isoformat()
else:
- enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window)
- enddate=enddate.isoformat()
+ time_to = starttime+int(time_window*1E9)
+# enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window)
+# enddate=enddate.isoformat()
#"2017-11-15 08:00:00"
try:
- times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
+# times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))
+ times=AKF_2.get_time_intervals(larchivepath,time_from,time_to)
except Exception as E:
- print('download_images_by_time_via_png: Error loading times from ',larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))
+# print('download_images_by_time_via_png: Error loading times from ',larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))
+ print('download_images_by_time_via_png: Error loading times from {0} between [{1}, {2}]'.format(larchivepath,time_from,time_to))
raise Warning(E)
return False,[0],[0]
@@ -870,6 +935,7 @@ if fastDL:
resultdict[order.index(i)]=[]
del resultdict
return success,np.array(times),images
+
class download_images_thread(threading.Thread):
def __init__(self, threadID, larchivepath,times,resultlist,version=0):
@@ -985,20 +1051,20 @@ if fastDL:
images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
self.resultlist.append([self.threadID,successes,time,images])
-def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,verbose=0,Test=True):
+def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,verbose=0,testmode=True):
"""
"""
- if Test:
- larchivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+"W7X/QSR07_FLIR/AEK51_raw"
+ if testmode:
+ larchivepath=testarchivepath+project+"/QSR07_FLIR/AEK51_raw"
else:
- larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
+ larchivepath=archivepath+project+"/QSR07_FLIR/AEK51_raw"
if type(time_window)==list:
tstart=time_window[0]
tstop=time_window[1]
if tstop<tstart:
- raise Exception("download_raw_FLIR_images_via_png: endtime before starttime")
+ raise Exception("download_raw_FLIR_images_via_png: t_stop before t_start")
starttime=int(t1-100+tstart*1e9)
- endtime=int(t1+tstop*1e9)
+ stoptime=int(t1+tstop*1e9)
# stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
# stdate=stdate.isoformat()
# enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)
@@ -1006,9 +1072,9 @@ def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,ver
else:
starttime=int(t1-100)
if time_window==0:
- endtime=t6
+ stoptime=t6
else:
- endtime=int((t1)+time_window*1e9)
+ stoptime=int((t1)+time_window*1e9)
# stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
# stdate=stdate.isoformat()
# if time_window==0:
@@ -1018,14 +1084,14 @@ def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,ver
# enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_window)
# enddate=enddate.isoformat()
#"2017-11-15 08:00:00"
- typ1_version=get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM",Test=Test,t_from=tstart)
+ typ1_version=get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM",testmode=testmode,t_from=tstart)
if typ1_version==None:
mode=1
else:
mode=2
if mode==1:
### okay the data is stored probably as different sets (so recorded with FLIR software)
- timess,versions=get_FLIR_Pset_times(starttime,endtime,version,Test=Test)#stdate,enddate,version,Test=Test)
+ timess,versions=get_FLIR_Pset_times(starttime,stoptime,version,testmode=testmode)#stdate,enddate,version,testmode=testmode)
if verbose>0:
print("download_raw_FLIR_images_via_png: found {0} frames in the different sets".format([np.shape(x) for x in timess]))
### we not know the times for all presets for the program, now we have to download them, if times is not empty
@@ -1052,19 +1118,19 @@ def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,ver
return exist,time,frames,mode,tsets,versions
-def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0,Test=True):
+def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0,testmode=True):
"""
"""
- if Test:
- larchivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+"W7X/QSR07_FLIR/AEK51_raw"
+ if testmode:
+ larchivepath=testarchivepath+project+"/QSR07_FLIR/AEK51_raw"
else:
- larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
+ larchivepath=archivepath+project+"/QSR07_FLIR/AEK51_raw"
if version==0:
versions=[]
- versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS0_DATASTREAM",Test=Test,t_from=stdate))
- versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS1_DATASTREAM",Test=Test,t_from=stdate))
- versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS2_DATASTREAM",Test=Test,t_from=stdate))
- versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS3_DATASTREAM",Test=Test,t_from=stdate))
+ versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS0_DATASTREAM",testmode=testmode,t_from=stdate))
+ versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS1_DATASTREAM",testmode=testmode,t_from=stdate))
+ versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS2_DATASTREAM",testmode=testmode,t_from=stdate))
+ versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS3_DATASTREAM",testmode=testmode,t_from=stdate))
else:
versions=[version,version,version,version]
timess=[]
@@ -1227,28 +1293,33 @@ def download_last_raw_image_by_time(larchivepath,starttime,stoptime,version=0):
print('download_last_raw_image_by_time: Error querrying {0}\n{1}'.format(url,e))
return False, [-1]
-def download_raw_parlog_by_program(port,program,version=0):
+def download_raw_parlog_by_program(port,program,version=0,testmode=False,verbose=0):
"""
"""
-# prog=AKF_1.get_program_from_PID(program)
- try:
- t_program = AKF_2.get_program_from_to(program)
- prog =AKF_2.get_program_list(t_program[0], t_program[1])
-# if prog[0]:
- starttime=prog[0]['trigger']['1'][0]
- stoptime=prog[0]['trigger']['6'][0]
- return download_raw_parlog_by_times(port,starttime,stoptime,version)
- except:
+## prog=AKF_1.get_program_from_PID(program)
+# try:
+# t_program = AKF_2.get_program_from_to(program)
+# prog =AKF_2.get_program_list(t_program[0], t_program[1])
+## if prog[0]:
+# starttime=prog[0]['trigger']['1'][0]
+# stoptime=prog[0]['trigger']['6'][0]
+ exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
+ return download_raw_parlog_by_times(port,starttime,stoptime,version,testmode=testmode)
+ else: #except:
print("download_raw_parlog_by_program: Error! cannot find the program")
return False,0,0
-def download_raw_parlog_by_times(port,starttime,stoptime,version=0):
+def download_raw_parlog_by_times(port,starttime,stoptime,version=0,testmode=False):
"""
"""
OP=IR_tools.get_OP_by_time(time_ns=stoptime)
if version==0:
- version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG",t_from=starttime)
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version)
+ version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG",t_from=starttime,testmode=testmode)
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version)
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version)
try:
res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
signal_list = json.loads(res.read().decode('utf-8'))
@@ -1264,28 +1335,34 @@ def download_raw_parlog_by_times(port,starttime,stoptime,version=0):
return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']
-def get_INFRATEC_filter_by_program(program,version=0):
+def get_INFRATEC_filter_by_program(program,port=50,version=0,testmode=False,verbose=0):
"""
"""
- prog=get_program_from_PID(program)
-# try:
-# t_program = AKF_2.get_program_from_to(program)
-# prog =AKF_2.get_program_list(t_program[0], t_program[1])
- if prog[0]:
- starttime=prog[1]['trigger']['1'][0]
- stoptime=prog[1]['trigger']['6'][0]
- return get_INFRATEC_filter_by_times(starttime,stoptime,50,version)
+# prog=get_program_from_PID(program)
+## try:
+## t_program = AKF_2.get_program_from_to(program)
+## prog =AKF_2.get_program_list(t_program[0], t_program[1])
+# if prog[0]:
+# starttime=prog[1]['trigger']['1'][0]
+# stoptime=prog[1]['trigger']['6'][0]
+ exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
+ return get_INFRATEC_filter_by_times(starttime,stoptime,50,version,testmode=testmode)
else:#except:
print("get_INFRATEC_filter_by_program: Error! cannot find the program")
return False,0,0
-def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0):
+def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0,testmode=False):
"""
"""
OP=IR_tools.get_OP_by_time(time_ns=starttime)
if version==0:
- version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG",t_from=starttime)
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version)
+ version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG",t_from=starttime,testmode=testmode)
+
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version)
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version)
try:
res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
signal_list = json.loads(res.read().decode('utf-8'))
@@ -1302,164 +1379,194 @@ def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0):
else:
return False, 0,-1
-def get_exposure_by_program(port,program,version=0):
+def get_exposure_by_program(port,program,version=0,testmode=False,verbose=0):
"""
"""
- prog=get_program_from_PID(program)
-# try:
-# t_program = AKF_2.get_program_from_to(program)
-# prog =AKF_2.get_program_list(t_program[0], t_program[1])
- if prog[0]:
- starttime=prog[1]['trigger']['1'][0]
- stoptime=prog[1]['trigger']['6'][0]
- return get_exposure_by_times(port,starttime,stoptime,version)
+# prog=get_program_from_PID(program)
+## try:
+## t_program = AKF_2.get_program_from_to(program)
+## prog =AKF_2.get_program_list(t_program[0], t_program[1])
+# if prog[0]:
+# starttime=prog[1]['trigger']['1'][0]
+# stoptime=prog[1]['trigger']['6'][0]
+ exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
+ return get_exposure_by_times(port,starttime,stoptime,version,testmode=testmode)
else:
# except Exception as E:
print("get_exposure_by_program: Error! cannot find the program, no exposure time available;")#, E)
return False,0,0
-def get_exposure_by_times(port,starttime,stoptime,version=0):
+def get_exposure_by_times(port,starttime,stoptime,version=0,testmode=False):
"""
"""
OP=IR_tools.get_OP_by_time(time_ns=stoptime)
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
- stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
+ stream=portpathdict[OP]["AEF"+str(port)]+"meta_"+"DATASTREAM"
if version==0:
- version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
+ version=get_latest_version(stream,t_from=starttime, testmode=testmode)
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/3/exposuretime"+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
-def get_camera_temp_by_program(port,program,version=0):
+def get_camera_temp_by_program(port,program,version=0,testmode=False,verbose=0):
"""
"""
-# prog=AKF_1.get_program_from_PID(program)
- try:
- t_program = AKF_2.get_program_from_to(program)
- prog =AKF_2.get_program_list(t_program[0], t_program[1])
-# if prog[0]:
- starttime=prog[0]['trigger']['1'][0]
- stoptime=prog[0]['trigger']['6'][0]
- return get_camera_temp_by_times(port,starttime,stoptime,version)
- except:
+## prog=AKF_1.get_program_from_PID(program)
+# try:
+# t_program = AKF_2.get_program_from_to(program)
+# prog =AKF_2.get_program_list(t_program[0], t_program[1])
+## if prog[0]:
+# starttime=prog[0]['trigger']['1'][0]
+# stoptime=prog[0]['trigger']['6'][0]
+ exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
+ return get_camera_temp_by_times(port,starttime,stoptime,version,testmode=testmode)
+ else: #except:
print("get_camera_temp_by_program: Error! cannot find the program")
return False,0,0
-def get_camera_temp_by_times(port,starttime,stoptime,version=0):
+def get_camera_temp_by_times(port,starttime,stoptime,version=0,testmode=False):
"""
"""
OP=IR_tools.get_OP_by_time(time_ns=stoptime)
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
- version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
+ version=get_latest_version(stream+"DATASTREAM",t_from=starttime,testmode=testmode)
dummy=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if dummy[0]:
return dummy
else:
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratempetarure/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
-def get_sensor_temp_by_program(port,program,version=0):
+def get_sensor_temp_by_program(port,program,version=0,testmode=False,verbose=0):
"""
"""
- prog=get_program_from_PID(program)
-# try:
-# t_program = AKF_2.get_program_from_to(program)
-# prog =AKF_2.get_program_list(t_program[0], t_program[1])
- if prog[0]:
- starttime=prog[1]['trigger']['1'][0]
- stoptime=prog[1]['trigger']['6'][0]
- return get_sensor_temp_by_times(port,starttime,stoptime,version)
+# prog=get_program_from_PID(program)
+## try:
+## t_program = AKF_2.get_program_from_to(program)
+## prog =AKF_2.get_program_list(t_program[0], t_program[1])
+# if prog[0]:
+# starttime=prog[1]['trigger']['1'][0]
+# stoptime=prog[1]['trigger']['6'][0]
+ exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
+ return get_sensor_temp_by_times(port,starttime,stoptime,version,testmode=testmode)
else:#except:
print("get_sensor_temp_by_program: Error! cannot find the program")
return False,0,0
-def get_sensor_temp_by_times(port,starttime,stoptime,version=0):
+def get_sensor_temp_by_times(port,starttime,stoptime,version=0,testmode=False):
"""
"""
OP=IR_tools.get_OP_by_time(time_ns=stoptime)
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
- version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
+ version=get_latest_version(stream+"DATASTREAM",t_from=starttime,testmode=testmode)
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
-def get_camera_framerate_by_program(port,program,version=0):
+def get_camera_framerate_by_program(port,program,version=0,testmode=False,verbose=0):
"""
"""
- prog=get_program_from_PID(program)
-# try:
-# t_program = AKF_2.get_program_from_to(program)
-# prog =AKF_2.get_program_list(t_program[0], t_program[1])
- if prog[0]:
- starttime=prog[1]['trigger']['1'][0]
- stoptime=prog[1]['trigger']['6'][0]
- return get_camera_framerate_by_times(port,starttime,stoptime,version)
+# prog=get_program_from_PID(program)
+## try:
+## t_program = AKF_2.get_program_from_to(program)
+## prog =AKF_2.get_program_list(t_program[0], t_program[1])
+# if prog[0]:
+# starttime=prog[1]['trigger']['1'][0]
+# stoptime=prog[1]['trigger']['6'][0]
+ exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
+ return get_camera_framerate_by_times(port,starttime,stoptime,version,testmode=testmode)
# except:
else:
print("get_camera_framerate_by_program: Error! cannot find the program")
return False,0,0
-def get_camera_framerate_by_times(port,starttime,stoptime,version=0):
+def get_camera_framerate_by_times(port,starttime,stoptime,version=0,testmode=False):
"""
"""
OP=IR_tools.get_OP_by_time(time_ns=stoptime)
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
- version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
+ version=get_latest_version(stream+"DATASTREAM",t_from=starttime,testmode=testmode)
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
-def get_frametype_by_program(port,program,version=0):
+def get_frametype_by_program(port,program,version=0,testmode=False,verbose=0):
"""
"""
-# prog=AKF_1.get_program_from_PID(program)
- try:
- t_program = AKF_2.get_program_from_to(program)
- prog =AKF_2.get_program_list(t_program[0], t_program[1])
-# if prog[0]:
- starttime=prog[0]['trigger']['1'][0]
- stoptime=prog[0]['trigger']['6'][0]
- return get_frametype_by_times(port,starttime,stoptime,version)
- except:
+## prog=AKF_1.get_program_from_PID(program)
+# try:
+# t_program = AKF_2.get_program_from_to(program)
+# prog =AKF_2.get_program_list(t_program[0], t_program[1])
+## if prog[0]:
+# starttime=prog[0]['trigger']['1'][0]
+# stoptime=prog[0]['trigger']['6'][0]
+ exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
+ return get_frametype_by_times(port,starttime,stoptime,version,testmode=testmode)
+ else: #except:
print("get_frametype_by_program: Error! cannot find the program")
return False,0,0
-def get_frametype_by_times(port,starttime,stoptime,version=0):
+def get_frametype_by_times(port,starttime,stoptime,version=0,testmode=False):
"""
"""
OP=IR_tools.get_OP_by_time(time_ns=starttime)
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
- version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
+ version=get_latest_version(stream+"DATASTREAM",t_from=starttime,testmode=testmode)
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/5/frametype/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
-def get_framenumber_by_program(port,program,version=0):
+def get_framenumber_by_program(port,program,version=0,testmode=False,verbose=0):
"""
"""
# prog=AKF_1.get_program_from_PID(program)
- try:
- t_program = AKF_2.get_program_from_to(program)
- prog =AKF_2.get_program_list(t_program[0], t_program[1])
-# if prog[0]:
- starttime=prog[0]['trigger']['1'][0]
- stoptime=prog[0]['trigger']['6'][0]
- return get_framenumber_by_times(port,starttime,stoptime,version)
- except:
+# try:
+# t_program = AKF_2.get_program_from_to(program)
+# prog =AKF_2.get_program_list(t_program[0], t_program[1])
+## if prog[0]:
+# starttime=prog[0]['trigger']['1'][0]
+# stoptime=prog[0]['trigger']['6'][0]
+ exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
+ return get_framenumber_by_times(port,starttime,stoptime,version,testmode=testmode)
+ else: #except:
print("get_framenumber_by_program: Error! cannot find the program")
return False,0,0
-def get_framenumber_by_times(port,starttime,stoptime,version=0):
+def get_framenumber_by_times(port,starttime,stoptime,version=0,testmode=False):
"""
"""
OP=IR_tools.get_OP_by_time(time_ns=starttime)
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
- version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
+ version=get_latest_version(stream+"DATASTREAM",t_from=starttime,testmode=testmode)
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/4/framenumber/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
-def get_average_divertor_TC_for_camtemp(port,starttime,endtime,part="all",TC=0):
+def get_average_divertor_TC_for_camtemp(port,starttime,stoptime,part="all",TC=0):
"""
"""
urlstart="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.115/"
@@ -1520,7 +1627,7 @@ def get_average_divertor_TC_for_camtemp(port,starttime,endtime,part="all",TC=0):
i+=1
url=urlstart+str(offset+i)+ext+"0"+str(i)+"_TC"+str(i%8+1)
try:
- result=read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
+ result=read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
# res.append(result)
if result[0]:
n+=1
@@ -1553,22 +1660,24 @@ def get_divertor_TC_coordinate(cameraport,TC=0):
raise Exception("get_divertor_TC_coordinate: unknown port!")
return TC_database[portnr][TC]
-def get_TC_data_for_port_by_program(port,program,part=0,version=0):
+def get_TC_data_for_port_by_program(port,program,part=0,version=0,verbose=0):
"""
"""
-# prog=AKF_1.get_program_from_PID(program)
- try:
- t_program = AKF_2.get_program_from_to(program)
- prog =AKF_2.get_program_list(t_program[0], t_program[1])
-# if prog[0]:
- starttime=prog[0]['trigger']['1'][0]
- stoptime=prog[0]['trigger']['6'][0]
+## prog=AKF_1.get_program_from_PID(program)
+# try:
+# t_program = AKF_2.get_program_from_to(program)
+# prog =AKF_2.get_program_list(t_program[0], t_program[1])
+## if prog[0]:
+# starttime=prog[0]['trigger']['1'][0]
+# stoptime=prog[0]['trigger']['6'][0]
+ exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode=False, verbose=verbose-1)
+ if exist:
return get_TC_data_for_port(port,starttime,stoptime,part,version)
- except:
+ else: #except:
print("get_TC_data_for_port_by_program: Error! cannot find the program")
return False,0,0
-def get_TC_data_for_port(port,starttime,endtime,part=0,version=0):
+def get_TC_data_for_port(port,starttime,stoptime,part=0,version=0):
"""
"""
urlstart="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.115/DataModuleDesc.19471_DATASTREAM/"
@@ -1584,7 +1693,7 @@ def get_TC_data_for_port(port,starttime,endtime,part=0,version=0):
for chi in range(len(thisTCchannel[0])):
url=urlstart+str(thisTCchannel[0][chi])+"/"+thisTCchannel[1][chi]
try:
- result=read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
+ result=read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if result[0]:
n+=1
time=result[1]
@@ -1599,7 +1708,7 @@ def get_TC_data_for_port(port,starttime,endtime,part=0,version=0):
try:
url=urlstart+str(thisTCchannel[0][part-1])+"/"+thisTCchannel[1][part-1]
try:
- result=read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
+ result=read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if result[0]:
n+=1
time=result[1]
@@ -1659,88 +1768,92 @@ def check_temperature_range(time,maxvalue_C,minvalue_C,port,exposure,cfilter=0):
def get_temp_from_raw_by_program_V1(portnr,program,time_window=0,emi=0.8,divertorpart="all",
- version=0,intervalSize=1E9,give_ERROR=False,verbose=0):
+ version=0,intervalSize=1E9,give_ERROR=False,testmode=False,verbose=0):
"""
"""
- background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=1,version=version,verbose=verbose-1)
-# prog=AKF_1.get_program_from_PID(program)
- try:
- t_program = AKF_2.get_program_from_to(program)
- prog = AKF_2.get_program_list(t_program[0], t_program[1])
-# if prog[0]:
-# t0=prog[1]['trigger']['0'][0]
- t1=prog[0]['trigger']['1'][0]
- t6=prog[0]['trigger']['6'][0]
- starttime=t1-10
- if type(time_window)==list:
- tstart=time_window[0]
- tstop=time_window[1]
- if tstop<tstart:
- raise Exception("get_temp_from_raw_by_program_V1: endtime before starttime")
- stoptime=int(starttime+tstop*1e9)
- starttime=int(starttime+tstart*1e9)
+
+## prog=AKF_1.get_program_from_PID(program)
+# try:
+# t_program = AKF_2.get_program_from_to(program)
+# prog = AKF_2.get_program_list(t_program[0], t_program[1])
+## if prog[0]:
+## t0=prog[1]['trigger']['0'][0]
+# t1=prog[0]['trigger']['1'][0]
+# t6=prog[0]['trigger']['6'][0]
+ exist, t0, t1, t6 = get_trigger_from_PID(program, portnr, testmode, verbose=verbose-1)
+ if not exist:
+ print("get_temp_from_raw_by_program_V1: Error! cannot find program")
+ return False,0,-1,False
+
+ background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0=t0,t1=t1,t6=t6,emissivity=emi,T_version=1,version=version,testmode=testmode,verbose=verbose-1)
+ starttime=t1-10
+ if type(time_window)==list:
+ tstart=time_window[0]
+ tstop=time_window[1]
+ if tstop<tstart:
+ raise Exception("get_temp_from_raw_by_program_V1: t_stop before t_start")
+ stoptime=int(starttime+tstop*1e9)
+ starttime=int(starttime+tstart*1e9)
+ else:
+ if time_window==0:
+ stoptime=t6
else:
- if time_window==0:
- stoptime=t6
- else:
- stoptime=int(starttime+time_window*1e9)
- success=True
+ stoptime=int(starttime+time_window*1e9)
+ success=True
+ if verbose>0:
+ print(datetime.datetime.now(),"get_temp_from_raw_by_program_V1: Start download of raw images")
+ if (stoptime-starttime)/intervalSize>1:
+ nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
if verbose>0:
- print(datetime.datetime.now(),"get_temp_from_raw_by_program_V1: Start download of raw images")
- if (stoptime-starttime)/intervalSize>1:
- nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
- if verbose>0:
- print("get_temp_from_raw_by_program_V1: time_window too large, splitting into {0} smaller fractions".format(nrinterv))
- intervalls=[]
- for i in range(nrinterv):
- intervalls.append(int(starttime-10+i*intervalSize))
- intervalls.append(stoptime)
- temperatureimages=[]
- times=[]
- for i in range(nrinterv):
- raw_dl=download_raw_images_by_times(portnr,intervalls[i],intervalls[i+1],version,intervalSize,verbose=verbose-1)
- print(datetime.datetime.now(),"get_temp_from_raw_by_program_V1: download of raw images part {0} of {1} finished".format(i+1, nrinterv))
- if raw_dl[0]:
- time=raw_dl[1]
- images=raw_dl[2]
- del raw_dl
-# images=images.swapaxes(1,2)
- images=[im.swapaxes(0,1) for im in images]
- sucess,images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,True,verbose=verbose-1)
- temperatureimages=temperatureimages+images
- times=times+time
- del time,images
- else:
- raise Exception("get_temp_from_raw_by_program_V1: cannot download the raw images")
- else:
- raw_dl=download_raw_images_by_times(portnr,starttime,stoptime,version,intervalSize,verbose=verbose-1)
+ print("get_temp_from_raw_by_program_V1: time_window too large, splitting into {0} smaller fractions".format(nrinterv))
+ intervalls=[]
+ for i in range(nrinterv):
+ intervalls.append(int(starttime-10+i*intervalSize))
+ intervalls.append(stoptime)
+ temperatureimages=[]
+ times=[]
+ for i in range(nrinterv):
+ raw_dl=download_raw_images_by_times(portnr,intervalls[i],intervalls[i+1],version,intervalSize,testmode=testmode,verbose=verbose-1)
+ print(datetime.datetime.now(),"get_temp_from_raw_by_program_V1: download of raw images part {0} of {1} finished".format(i+1, nrinterv))
if raw_dl[0]:
- times=raw_dl[1]
- temperatureimages=raw_dl[2]
+ time=raw_dl[1]
+ images=raw_dl[2]
del raw_dl
-# temperatureimages=temperatureimages.swapaxes(1,2)
- temperatureimages=[im.swapaxes(0,1) for im in temperatureimages]
- if give_ERROR:
- success,temperatureimages,error_images=apply_calib_on_raw(temperatureimages,background,LUT,refT,gain,offset,gain_error,offset_error,True,give_ERROR,verbose=verbose-1)
- else:
- success,temperatureimages=apply_calib_on_raw(temperatureimages,background,LUT,refT,gain,offset,gain_error,offset_error,True,give_ERROR,verbose=verbose-1)
+# images=images.swapaxes(1,2)
+ images=[im.swapaxes(0,1) for im in images]
+ sucess,images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,True,verbose=verbose-1)
+ temperatureimages=temperatureimages+images
+ times=times+time
+ del time,images
else:
raise Exception("get_temp_from_raw_by_program_V1: cannot download the raw images")
- valid=True
- for i in range(len(temperatureimages)):
- try:
- check_temperature_range(times[0],np.max(temperatureimages)-273.15,np.min(temperatureimages)-273.15,portnr,t_exp,cfilter)
- except Exception as E:
- print("get_temp_from_raw_by_program_V1: Error! ",E)
- valid=False
- i=len(temperatureimages)
- if give_ERROR:
- return success,times,temperatureimages,valid,error_images
+ else:
+ raw_dl=download_raw_images_by_times(portnr,starttime,stoptime,version,intervalSize,testmode=testmode,verbose=verbose-1)
+ if raw_dl[0]:
+ times=raw_dl[1]
+ temperatureimages=raw_dl[2]
+ del raw_dl
+# temperatureimages=temperatureimages.swapaxes(1,2)
+ temperatureimages=[im.swapaxes(0,1) for im in temperatureimages]
+ if give_ERROR:
+ success,temperatureimages,error_images=apply_calib_on_raw(temperatureimages,background,LUT,refT,gain,offset,gain_error,offset_error,True,give_ERROR,verbose=verbose-1)
+ else:
+ success,temperatureimages=apply_calib_on_raw(temperatureimages,background,LUT,refT,gain,offset,gain_error,offset_error,True,give_ERROR,verbose=verbose-1)
else:
- return success,times,temperatureimages,valid
- except:
- print("get_temp_from_raw_by_program_V1: Error! cannot find program")
- return False,0,-1,False
+ raise Exception("get_temp_from_raw_by_program_V1: cannot download the raw images")
+ valid=True
+ for i in range(len(temperatureimages)):
+ try:
+ check_temperature_range(times[0],np.max(temperatureimages)-273.15,np.min(temperatureimages)-273.15,portnr,t_exp,cfilter)
+ except Exception as E:
+ print("get_temp_from_raw_by_program_V1: Error! ",E)
+ valid=False
+ i=len(temperatureimages)
+ if give_ERROR:
+ return success,times,temperatureimages,valid,error_images
+ else:
+ return success,times,temperatureimages,valid
+
def get_temp_from_raw_by_program_V2(portnr,program,time_window=0,emi=0.8,version=0,
threads=1,give_ERROR=False,use_firstframe_as_background=False,
@@ -1756,200 +1869,200 @@ def get_temp_from_raw_by_program_V2(portnr,program,time_window=0,emi=0.8,version
def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=2,
version=0,threads=1,give_ERROR=False,
use_firstframe_as_background=False,back_emissivity=0.8,
- verbose=0,Test=False,framerate='max'):
+ verbose=0,testmode=False,framerate='max'):
"""
"""
FLIR=False
- prog=get_program_from_PID(program)
- if prog[0]:
- t1=prog[1]['trigger']['1'][0]
- t6=prog[1]['trigger']['6'][0]
- success=True
- OP=IR_tools.get_OP_by_time(time_ns=t1)
- if type(portnr) is str:
- try:
- camera=portcamdict[OP][portnr]
- if camera.split("_")[0]=='FLIR':
- FLIR=True
+ trigger_exist, t0, t1, t6 = get_trigger_from_PID(program, portnr, testmode, verbose=verbose-1)
+ if not trigger_exist:
+ raise Exception("get_temp_from_raw_by_program: unknown Program")
+
+ success=True
+ OP=IR_tools.get_OP_by_time(time_ns=t1)
+ if type(portnr) is str:
+ try:
+ camera=portcamdict[OP][portnr]
+ if camera.split("_")[0]=='FLIR':
+ FLIR=True
+ else:
+ FLIR=False
+ portnr=int(portnr.split("AEF")[1])
+ except Exception as E:
+ raise Exception("get_temp_from_raw_by_program: unknown Port!"+E)
+ elif isinstance(portnr, (int, np.integer, float, np.float)):
+ FLIR=False
+ portnr=int(portnr)
+ else:
+ raise Exception("get_temp_from_raw_by_program: the given port is neither a number or a valid String!")
+
+ if FLIR:
+ # --- Glens FLIR camera ---
+ #
+ # get raw data
+ exist,time,frames,mode,tsets,versions=download_raw_FLIR_images_via_png(t1=t1,t6=t6,time_window=time_window,version=version,threads=threads,verbose=verbose-1,testmode=testmode)
+ if not exist:
+ print("get_temp_from_raw_by_program: Error! FLIR data not found")
+ return False,[0],[0],[]
+ # get calibration
+ if mode==1:
+ Psets=IR_tools.check_dublicates(tsets)
+ Radi_Co=[]
+ Temp_Co=[]
+ if len(Psets)==1:
+ # single exposure time case
+ PS=Psets[0]
+ if testmode:
+ parlog=read_restdb_old(testarchivepath+"W7X/QSR07_FLIR/AEK51_raw_PS"+str(PS)+"_PARLOG/V"+str(versions[PS])+"/_signal.json?from="+str(t1-10)+"&upto="+str(t6+20))
else:
- FLIR=False
- portnr=int(portnr.split("AEF")[1])
- goon=True
- except Exception as E:
- raise Exception("get_temp_from_raw_by_program: unknown Port!"+E)
- elif isinstance(portnr, (int, np.integer, float, np.float)):
- FLIR=False
- goon=True
- portnr=int(portnr)
- else:
- goon=False
- if not goon:
- raise Exception("get_temp_from_raw_by_program: the given port is neither a number or a valid String!")
- else:
- if FLIR:
- exist,time,frames,mode,tsets,versions=download_raw_FLIR_images_via_png(t1=t1,t6=t6,time_window=time_window,version=version,threads=threads,verbose=verbose-1,Test=Test)
- if exist:
- if mode==1:
- Psets=IR_tools.check_dublicates(tsets)
- Radi_Co=[]
- Temp_Co=[]
- if len(Psets)==1:
- single=True
- else:
- single=False
- if single:
- PS=Psets[0]
- if Test:
- parlog=read_restdb_old("http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+"W7X/QSR07_FLIR/AEK51_raw_PS"+str(PS)+"_PARLOG/V"+str(versions[PS])+"/_signal.json?from="+str(t1-10)+"&upto="+str(t6+20))
- else:
- parlog=read_restdb_old(archivepath+"W7X/QSR07_FLIR/AEK51_raw_PS"+str(PS)+"_PARLOG/V"+str(versions[PS])+"/_signal.json?from="+str(t1-10)+"&upto="+str(t6+20))
- tsets=[0 for i in range(len(frames))]
- if parlog[0]:
- pari=parlog[2][0]['meta-data']
- ipi=0
- for inti in range(4):
- try:
- dummy=pari['ITime_'+str(inti)]
- ipi=inti
- break
- except:
- ipi=-1
- c1=[float(pari['Coeff0_'+str(ipi)].split("\n")[0]),float(pari['Coeff1_'+str(ipi)].split("\n")[0])]
- c2=[float(pari['TempCoeff'+str(x)+'_'+str(ipi)].split("\n")[0]) for x in range(7)]
- Radi_Co.append(c1)
- Temp_Co.append(c2)
- else:
- Radi_Co.append([])
- Temp_Co.append([])
- else:
- for PS in range(4):
- if PS in Psets:
- if Test:
- parlog=read_restdb_old("http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+"W7X/QSR07_FLIR/AEK51_raw_PS"+str(PS)+"_PARLOG/V"+str(versions[PS])+"/_signal.json?from="+str(t1-10)+"&upto="+str(t6+20))
- else:
- parlog=read_restdb_old(archivepath+"W7X/QSR07_FLIR/AEK51_raw_PS"+str(PS)+"_PARLOG/V"+str(versions[PS])+"/_signal.json?from="+str(t1-10)+"&upto="+str(t6+20))
- if parlog[0]:
- pari=parlog[2][0]['meta-data']
- c1=[float(pari['Coeff0_'+str(PS)].split("\n")[0]),float(pari['Coeff1_'+str(PS)].split("\n")[0])]
- c2=[float(pari['TempCoeff'+str(x)+'_'+str(PS)].split("\n")[0]) for x in range(7)]
- Radi_Co.append(c1)
- Temp_Co.append(c2)
- else:
- Radi_Co.append([])
- Temp_Co.append([])
- else:
- Radi_Co.append([])
- Temp_Co.append([])
- ### got the raw, knowing which frame needs which calibration, got the calibration, now we have to transform it
- if verbose>0:
- print(datetime.datetime.now(),"get_temp_from_raw_by_program: Converting the raw data into temperature, number of frames:",len(frames))
- for i in range(len(frames)):
- try:
- frames[i]=Radi_Co[tsets[i]][0]+frames[i]*Radi_Co[tsets[i]][1]
- dummy=frames[i]
- temp=Temp_Co[tsets[i]][0]+Temp_Co[tsets[i]][1]*dummy
- for expo in range(2,7):
- temp=temp+Temp_Co[tsets[i]][expo]*np.power(dummy,expo)
- frames[i]=temp+273.15
- except Exception as E:
- print('get_temp_from_raw_by_program: Error in frame {0}! {1}'.format(i,E))
- print(' (len(tsets) {0},Radi_Co[tsets[i]] {1},Temp_Co[tsets[i]] {2})'.format(len(tsets),Radi_Co[tsets[i]],Temp_Co[tsets[i]]))
- if verbose>0:
- print(datetime.datetime.now(),"get_temp_from_raw_by_program: temperature frames done")
- return exist,time,frames,tsets
- elif mode==2:
-
- raise Exception("get_temp_from_raw_by_program: not implemented in mode 2")
- else:
- raise Exception("get_temp_from_raw_by_program: FLIR evaluation, This case should not be possible.")
+ parlog=read_restdb_old(archivepath+"W7X/QSR07_FLIR/AEK51_raw_PS"+str(PS)+"_PARLOG/V"+str(versions[PS])+"/_signal.json?from="+str(t1-10)+"&upto="+str(t6+20))
+ tsets=[0 for i in range(len(frames))]
+ if parlog[0]:
+ pari=parlog[2][0]['meta-data']
+ ipi=0
+ for inti in range(4):
+ try:
+ dummy=pari['ITime_'+str(inti)]
+ ipi=inti
+ break
+ except:
+ ipi=-1
+ c1=[float(pari['Coeff0_'+str(ipi)].split("\n")[0]),float(pari['Coeff1_'+str(ipi)].split("\n")[0])]
+ c2=[float(pari['TempCoeff'+str(x)+'_'+str(ipi)].split("\n")[0]) for x in range(7)]
+ Radi_Co.append(c1)
+ Temp_Co.append(c2)
else:
- print("get_temp_from_raw_by_program: Error! FLIR data not found")
- return False,[0],[0],[]
+ Radi_Co.append([])
+ Temp_Co.append([])
else:
- cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,back_emissivity=back_emissivity,verbose=verbose-1)
- if not cexist:
- if verbose>0:
- print("get_temp_from_raw_by_program: Unable to load the complete calibration data for",program," please confirm that this data was uploaded")
- return False,[0],[0],False
- else:
- if version==0:
- version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM",program=program)
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
- if verbose>0:
- print(datetime.datetime.now(),"get_temp_from_raw_by_program: raw download start")
- if fastDL:
- exist,time,frames=download_images_by_time_via_png(larchivepath,starttime=t1,stoptime=t6,time_window=time_window,version=version,threads=threads,verbose=verbose-1,framerate=framerate)
- else:
- exist,time,frames=download_raw_images_by_program(portnr,program,time_window,version,verbose=verbose-1)
- if exist:
- frames=[im.swapaxes(0,1) for im in frames]
+ # multi exposure time case
+ for PS in range(4):
+ if PS in Psets:
+ if testmode:
+ parlog=read_restdb_old(testarchivepath+"W7X/QSR07_FLIR/AEK51_raw_PS"+str(PS)+"_PARLOG/V"+str(versions[PS])+"/_signal.json?from="+str(t1-10)+"&upto="+str(t6+20))
else:
- raise Exception("get_temp_from_raw_by_program: no data found")
- if not exist:
- raise Warning("get_temp_from_raw_by_program: data not found in database!")
- return False,[0],[0],False
- else:
- if verbose>0:
- print(datetime.datetime.now(),"get_temp_from_raw_by_program: raw download finished")
- FOV=get_FOV_mask(portnr)
- if use_firstframe_as_background:
- bim=apply_NUC([frames[0]],gain,offset)
- background=get_average_background_recangle(portnr,bim[0])
- if give_ERROR:
- success,frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
- else:
- success,frames=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
- if verbose>0:
- print(datetime.datetime.now(),"get_temp_from_raw_by_program: Temperature calculation done")
- if not success:
- if verbose>0:
- print("calculation of the temperature failed, calibration could not be applied")
- return False,time,frames,False
+ parlog=read_restdb_old(archivepath+"W7X/QSR07_FLIR/AEK51_raw_PS"+str(PS)+"_PARLOG/V"+str(versions[PS])+"/_signal.json?from="+str(t1-10)+"&upto="+str(t6+20))
+ if parlog[0]:
+ pari=parlog[2][0]['meta-data']
+ c1=[float(pari['Coeff0_'+str(PS)].split("\n")[0]),float(pari['Coeff1_'+str(PS)].split("\n")[0])]
+ c2=[float(pari['TempCoeff'+str(x)+'_'+str(PS)].split("\n")[0]) for x in range(7)]
+ Radi_Co.append(c1)
+ Temp_Co.append(c2)
else:
-
- for i in range(len(frames)):
- frames[i]=(frames[i]*FOV).astype(np.float32)
-
-# if portnr!=50:
- if verbose>0:
- print(datetime.datetime.now(),"get_temp_from_raw_by_program: correcting bad pixels")
- frames=IR_tools.correct_images(frames,badpixels)
- if verbose>0:
- print(datetime.datetime.now(),"get_temp_from_raw_by_program: checking temperaturerange")
- valid=True
- # for i in range(len(frames)):
- # try:
- # check_temperature_range(time[0],np.max(frames)-273.15,np.min(frames[:][np.nonzero(frames[i])])-273.15,portnr,t_exp,cfilter)
- # except Exception as E:
- # print(E)
- # valid=False
- # i=len(frames)
- date=int(program.split(".")[0])
- if date>20180712 and date<20180720:
-# prog=get_program_from_PID(program)
- if prog[0]:
- # t0time=prog[1]['trigger']['0'][0]
- starttime=t1#prog[1]['trigger']['1'][0]
- toff=time[0]-starttime
- if toff>10e6:
- print("get_temp_from_raw_by_program: time offset of {0}ns detected, correcting".format(toff))
- time[:]=time[:]-toff
- if give_ERROR:
- return exist,time,frames,valid,error_images
- else:
- return exist,time,frames,valid
+ Radi_Co.append([])
+ Temp_Co.append([])
+ else:
+ Radi_Co.append([])
+ Temp_Co.append([])
+ # got the raw, knowing which frame needs which calibration, got the calibration, now we have to transform it
+ if verbose>0:
+ print(datetime.datetime.now(),"get_temp_from_raw_by_program: Converting the raw data into temperature, number of frames:",len(frames))
+ for i in range(len(frames)):
+ try:
+ frames[i]=Radi_Co[tsets[i]][0]+frames[i]*Radi_Co[tsets[i]][1]
+ dummy=frames[i]
+ temp=Temp_Co[tsets[i]][0]+Temp_Co[tsets[i]][1]*dummy
+ for expo in range(2,7):
+ temp=temp+Temp_Co[tsets[i]][expo]*np.power(dummy,expo)
+ frames[i]=temp+273.15
+ except Exception as E:
+ print('get_temp_from_raw_by_program: Error in frame {0}! {1}'.format(i,E))
+ print(' (len(tsets) {0},Radi_Co[tsets[i]] {1},Temp_Co[tsets[i]] {2})'.format(len(tsets),Radi_Co[tsets[i]],Temp_Co[tsets[i]]))
+ if verbose>0:
+ print(datetime.datetime.now(),"get_temp_from_raw_by_program: temperature frames done")
+ return exist,time,frames,tsets
+ elif mode==2:
+ raise Exception("get_temp_from_raw_by_program: not implemented in mode 2")
+ else:
+ raise Exception("get_temp_from_raw_by_program: FLIR evaluation, This case should not be possible.")
+
else:
- raise Exception("get_temp_from_raw_by_program: unknown Program")
+ # --- IRcam case ---
+ #
+ # get calibration data
+ cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0=t0,t1=t1,t6=t6,emissivity=emi,T_version=T_version,version=version,back_emissivity=back_emissivity,testmode=testmode,verbose=verbose-1)
+ if not cexist:
+ if verbose>0:
+ print("get_temp_from_raw_by_program: Unable to load the complete calibration data for",program," please confirm that this data was uploaded")
+ return False,[0],[0],False
+ # get raw data from latest or requested version
+ if version==0:
+ version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM",testmode=testmode,program=program)
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
+ if verbose>0:
+ print(datetime.datetime.now(),"get_temp_from_raw_by_program: raw download start")
+ if fastDL:
+ exist,time,frames=download_images_by_time_via_png(larchivepath,starttime=t1,stoptime=t6,time_window=time_window,version=version,threads=threads,verbose=verbose-1,framerate=framerate)
+ else:
+ exist,time,frames=download_raw_images_by_program(portnr,program,time_window,version,testmode=testmode,verbose=verbose-1)
+ if exist:
+ frames=[im.swapaxes(0,1) for im in frames]
+ if not exist:
+ raise Warning("get_temp_from_raw_by_program: data not found in database!")
+ return False,[0],[0],False
+ if verbose>0:
+ print(datetime.datetime.now(),"get_temp_from_raw_by_program: raw download finished")
+ # calibrate frames
+ FOV=get_FOV_mask(portnr)
+ if use_firstframe_as_background:
+ bim=apply_NUC([frames[0]],gain,offset)
+ background=get_average_background_recangle(portnr,bim[0])
+ if give_ERROR:
+ success,frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
+ else:
+ success,frames=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
+ if verbose>0:
+ print(datetime.datetime.now(),"get_temp_from_raw_by_program: Temperature calculation done")
+ if not success:
+ if verbose>0:
+ print("calculation of the temperature failed, calibration could not be applied")
+ return False,time,frames,False
+
+ # crop to FOV
+ for i in range(len(frames)):
+ frames[i]=(frames[i]*FOV).astype(np.float32)
+ # correct bad pixels
+ if verbose>0:
+ print(datetime.datetime.now(),"get_temp_from_raw_by_program: correcting bad pixels")
+ frames=IR_tools.correct_images(frames,badpixels)
+
+ # TODO: check temperature range for saturation and mark frames as not valid
+ #
+# if verbose>0:
+# print(datetime.datetime.now(),"get_temp_from_raw_by_program: checking temperaturerange")
+ valid=True
+# for i in range(len(frames)):
+# try:
+# check_temperature_range(time[0],np.max(frames)-273.15,np.min(frames[:][np.nonzero(frames[i])])-273.15,portnr,t_exp,cfilter)
+# except Exception as E:
+# print(E)
+# valid=False
+# i=len(frames)
+ # check for time vector offsets
+ date=int(program.split(".")[0])
+ if date>20180712 and date<20180720:
+ toff=time[0]-t1
+ if toff>10e6:
+ print("get_temp_from_raw_by_program: time offset of {0}ns detected, correcting".format(toff))
+ time[:]=time[:]-toff
+ if give_ERROR:
+ return exist,time,frames,valid,error_images
+ else:
+ return exist,time,frames,valid
+
def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8,
T_version=2,version=0,threads=1,
- give_ERROR=False,check_range=True,verbose=0):
+ give_ERROR=False,check_range=True,
+ testmode=False,verbose=0):
"""
"""
if threads==1 or not fastDL:
return get_temp_from_raw_by_program(portnr,program,time_window,emi,
T_version,version,threads,give_ERROR,
- verbose=verbose-1)
+ testmode=testmode, verbose=verbose-1)
else:
if type(portnr) is str:
try:
@@ -1970,22 +2083,27 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.
else:
goon=False
if goon and not FLIR:
- exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,verbose=verbose-1)
- FOV=get_FOV_mask(portnr)
- prog=get_program_from_PID(program)
- if prog[0]:
- starttime=prog[1]['trigger']['1'][0]
- stoptime=prog[1]['trigger']['6'][0]
+# prog=get_program_from_PID(program)
+# if prog[0]:
+# starttime=prog[1]['trigger']['1'][0]
+# stoptime=prog[1]['trigger']['6'][0]
+ exist, t0, starttime, stoptime = get_trigger_from_PID(program, portnr, testmode, verbose=verbose-1)
+ if exist:
+ exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0=t0,t1=starttime,t6=stoptime,emissivity=emi,T_version=T_version,version=version,testmode=testmode,verbose=verbose-1)
+ FOV=get_FOV_mask(portnr)
success=True
OP=IR_tools.get_OP_by_time(time_ns=starttime)
if version==0:
- version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM",program=program)
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
+ version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM",program=program,testmode=testmode)
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
+ else:
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
if type(time_window)==list:
tstart=time_window[0]
tstop=time_window[1]
if tstop<tstart:
- raise Exception("get_temp_from_raw_by_program_fullthreads: endtime before starttime")
+ raise Exception("get_temp_from_raw_by_program_fullthreads: t_stop before t_start")
stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
stdate=stdate.isoformat()
enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
@@ -2064,16 +2182,18 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.
def get_nuced_raw_by_program(portnr,program,time_window=0,version=0,
emi=1,T_version=2,threads=1,
- give_ERROR=False,verbose=0):
+ give_ERROR=False,testmode=False,verbose=0):
"""
"""
- exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,verbose=verbose-1)
+ # get calibration data
+ exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,testmode=testmode,verbose=verbose-1)
if verbose>0:
print(datetime.datetime.now(),"get_nuced_raw_by_program: raw download start")
if fastDL:
- exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_window,version,threads,verbose=verbose-1)
+ exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_window,version,
+ threads,testmode=testmode,verbose=verbose-1)
else:
- exist,time,frames=download_raw_images_by_program(portnr,program,time_window,version,verbose=verbose-1)
+ exist,time,frames=download_raw_images_by_program(portnr,program,time_window,version,testmode=testmode,verbose=verbose-1)
if exist:
frames=[im.swapaxes(0,1) for im in frames]
else:
@@ -2102,97 +2222,106 @@ def get_nuced_raw_by_program(portnr,program,time_window=0,version=0,
def get_nuced_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8,
T_version=2,version=1,threads=1,
- give_ERROR=False,check_range=True,verbose=0):
- """
- """
- prog=get_program_from_PID(program)
- if prog[0]:
- starttime=prog[1]['trigger']['1'][0]
- stoptime=prog[1]['trigger']['6'][0]
- if threads==1 or not fastDL:
- return get_nuced_raw_by_program(portnr,program,
- time_window=time_window,emi=emi,
- T_version=T_version,version=version,
- threads=threads,give_ERROR=give_ERROR,
- verbose=verbose-1)
+ give_ERROR=False,check_range=True,
+ testmode=False,verbose=0):
+ """
+ """
+# prog=get_program_from_PID(program)
+# if prog[0]:
+# starttime=prog[1]['trigger']['1'][0]
+# stoptime=prog[1]['trigger']['6'][0]
+ exist, t0, starttime, stoptime = get_trigger_from_PID(program, portnr, testmode, verbose=verbose-1)
+ if not exist:
+ print('get_nuced_raw_by_program_fullthreads: Error! no trigger timestamps found')
+ return False, None, None, None
+
+ if threads==1 or not fastDL:
+ return get_nuced_raw_by_program(portnr,program,
+ time_window=time_window,emi=emi,
+ T_version=T_version,version=version,
+ threads=threads,give_ERROR=give_ERROR,
+ testmode=testmode, verbose=verbose-1)
+ else:
+ exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0=t0,t1=starttime,t6=stoptime,emissivity=emi,T_version=T_version,version=version,testmode=testmode,verbose=verbose-1)
+ FOV=get_FOV_mask(portnr)
+ success=True
+ OP=IR_tools.get_OP_by_time(time_ns=starttime)
+ if verbose>0:
+ print('get_nuced_raw_by_program_fullthreads: prepare loading images by threads')
+ if version==0:
+ version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM",t_from=starttime,testmode=testmode)
+ if testmode:
+ larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
else:
- exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,verbose=verbose-1)
- FOV=get_FOV_mask(portnr)
- success=True
- OP=IR_tools.get_OP_by_time(time_ns=starttime)
- if verbose>0:
- print('get_nuced_raw_by_program_fullthreads: prepare loading images by threads')
- if version==0:
- version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM",t_from=starttime)
- larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
- if type(time_window)==list:
- tstart=time_window[0]
- tstop=time_window[1]
- if tstop<tstart:
- raise Exception("get_nuced_raw_by_program_fullthreads: endtime before starttime")
- stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
- stdate=stdate.isoformat()
- enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
+ larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
+ if type(time_window)==list:
+ tstart=time_window[0]
+ tstop=time_window[1]
+ if tstop<tstart:
+ raise Exception("get_nuced_raw_by_program_fullthreads: t_stop before t_start")
+ stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
+ stdate=stdate.isoformat()
+ enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)
+ enddate=enddate.isoformat()
+ else:
+ stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
+ stdate=stdate.isoformat()
+ if time_window==0:
+ enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)
enddate=enddate.isoformat()
else:
- stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
- stdate=stdate.isoformat()
- if time_window==0:
- enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)
- enddate=enddate.isoformat()
- else:
- enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window)
- enddate=enddate.isoformat()
- times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
- lnt=len(times)
- tim=[]
- for i in range(lnt):
- tim.append(times[lnt-1-i][0])
- intervalls=[]
- intervalSize=int(lnt/threads)
- for i in range(threads):
- intervalls.append(int(i*intervalSize))
- intervalls.append(lnt)
- jobs = []
- resultdict = []
- for i in range(threads):
- if verbose>0:
- print("get_nuced_raw_by_program_fullthreads: Start Thread ",i+1)
-# p = multiprocessing.Process(target=NUC_raw_thread, args=(portnr,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,))
- p = NUC_raw_thread(larchivepath,tim[intervalls[i]:intervalls[i+1]],
- resultdict,i,version,background,LUT,refT,
- gain,offset,gain_error,offset_error,give_ERROR,
- FOV,badpixels,verbose=verbose-1)
- jobs.append(p)
- p.start()
- for p in jobs:
- p.join()
+ enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window)
+ enddate=enddate.isoformat()
+ times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
+ lnt=len(times)
+ tim=[]
+ for i in range(lnt):
+ tim.append(times[lnt-1-i][0])
+ intervalls=[]
+ intervalSize=int(lnt/threads)
+ for i in range(threads):
+ intervalls.append(int(i*intervalSize))
+ intervalls.append(lnt)
+ jobs = []
+ resultdict = []
+ for i in range(threads):
if verbose>0:
- print("get_nuced_raw_by_program_fullthreads: all threads are done")
- order=[]
- for ele in resultdict:
- order.append(ele[0])
- if len(np.where(np.asarray(ele[1])==False)[0])>0:
- success=False
- times=[]#np.array(resultdict[order.index(0)][2])
- images=[]#np.array(resultdict[order.index(0)][3])
- for i in range(threads):
- images=images+resultdict[order.index(i)][3]
- times=times+resultdict[order.index(i)][2]
- del resultdict
- valid=True
+ print("get_nuced_raw_by_program_fullthreads: Start Thread ",i+1)
+# p = multiprocessing.Process(target=NUC_raw_thread, args=(portnr,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,))
+ p = NUC_raw_thread(larchivepath,tim[intervalls[i]:intervalls[i+1]],
+ resultdict,i,version,background,LUT,refT,
+ gain,offset,gain_error,offset_error,give_ERROR,
+ FOV,badpixels,verbose=verbose-1)
+ jobs.append(p)
+ p.start()
+ for p in jobs:
+ p.join()
+ if verbose>0:
+ print("get_nuced_raw_by_program_fullthreads: all threads are done")
+ order=[]
+ for ele in resultdict:
+ order.append(ele[0])
+ if len(np.where(np.asarray(ele[1])==False)[0])>0:
+ success=False
+ times=[]#np.array(resultdict[order.index(0)][2])
+ images=[]#np.array(resultdict[order.index(0)][3])
+ for i in range(threads):
+ images=images+resultdict[order.index(i)][3]
+ times=times+resultdict[order.index(i)][2]
+ del resultdict
+ valid=True
# else:
# success=False
# times=[0]
# images=[0]
# valid=False
# error_images=[0]
- print(larchivepath)
-
- if give_ERROR:
- return success,np.array(times),images,valid#,error_images
- else:
- return success,np.array(times),images,valid
+# print(larchivepath)
+
+ if give_ERROR:
+ return success,np.array(times),images,valid#,error_images
+ else:
+ return success,np.array(times),images,valid
def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_error=0,
offset_error=0,fullbackground=False,give_ERROR=False,verbose=0):
@@ -2278,12 +2407,12 @@ def get_cooling_water_temperature_by_time(time_ns):
"""
"""
starttime=int(time_ns-3e9)
- endtime=int(time_ns+3e9)
+ stoptime=int(time_ns+3e9)
outlet_url="95/Ist%20Av06_ABK10_CT006"
inlet_url="94/Ist%20Av06_ABK10_CT003"
url_base="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.85/DataModuleDesc.188_DATASTREAM/"
try:
- result_in=read_restdb(url_base+inlet_url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
+ result_in=read_restdb(url_base+inlet_url+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if result_in[0]:
inlet=np.mean(np.array(result_in[2]))
else:
@@ -2294,7 +2423,7 @@ def get_cooling_water_temperature_by_time(time_ns):
inlet=0
result_in=[False,0,0]
try:
- result_out=read_restdb(url_base+outlet_url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
+ result_out=read_restdb(url_base+outlet_url+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if result_out[0]:
outlet=np.mean(np.array(result_out[2]))
else:
@@ -2307,7 +2436,7 @@ def get_cooling_water_temperature_by_time(time_ns):
return result_in[0] and result_out[0],inlet,outlet
def get_calib_data(port,program=None,t0=None,t1=None,t6=None,emissivity=0.8,
- T_version=2,version=0,back_emissivity=0.82,verbose=0):
+ T_version=2,version=0,back_emissivity=0.82,testmode=False,verbose=0):
"""
get_calib_data
return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
@@ -2322,23 +2451,19 @@ def get_calib_data(port,program=None,t0=None,t1=None,t6=None,emissivity=0.8,
raise Exception('get_calib_data: ERROR! Missing time definition!')
print(' function requires either a program ID or the trigger times t0, t1 and t6')
# get start and end times from program ID
- prog=get_program_from_PID(program)
- if prog[0]:
- t0=prog[1]['trigger']['0'][0]
- t1=prog[1]['trigger']['1'][0]
- t6=prog[1]['trigger']['6'][0]
- else:
+ exist, t0, t1, t6 = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if not exist:
raise Exception('get_calib_data: ERROR! program {0} not found!'.format(program))
-
+
+ # determin camera in this port and OP
OP=IR_tools.get_OP_by_time(time_ns=t0)
exist=True
-
if verbose>0:
print('get_calib_data: loading data for camera {1} in {0}'.format(OP,port))
Camera=portcamdict[OP]['AEF'+str(port)]
if Camera.split("_")[0]=="Infratec":
# cf=get_INFRATEC_filter_by_program(program,version)
- cf=get_INFRATEC_filter_by_times(starttime=t1,stoptime=t6)
+ cf=get_INFRATEC_filter_by_times(starttime=t1,stoptime=t6,testmode=testmode)
if cf[0]:
cfilter=cf[2]
else:
@@ -2347,138 +2472,149 @@ def get_calib_data(port,program=None,t0=None,t1=None,t6=None,emissivity=0.8,
return False,0,0,0,0,0,0,0,0,0,0
else:
cfilter=0
- expo_DL=get_exposure_by_times(port,t1,t6,version)
+
+ # get exposure time of first frame
+ expo_DL=get_exposure_by_times(port,t1,t6,version,testmode)
if expo_DL[0]:
t_exp=int(expo_DL[2][0])
del expo_DL
- time=int(TimeToNs([2017,9,26],[8,0,0,0]))
- LUT_DL=download_LUT(port,time,t_exp,emissivity,cfilter,version, verbose=verbose-1)
- if LUT_DL[0]:
- LUT=LUT_DL[1]
- del LUT_DL
- if cfilter==0 and not Camera.split("_")[0]=="Infratec":
- if verbose>0:
- print('get_calib_data: getting NUC arrays')
- NUC_DL=get_NUC_by_times(port,t0,t1,t_exp,version,verbose=verbose-1)#download_NUC_by_times(port,t0,t1,t_exp,version)
- if NUC_DL[0]==False:
- raise Warning("get_calib_data: NUC was not found")
- return False,0,LUT,0,0,0,0,t_exp,cfilter,0,0
- else:
- # extract NUC elements
- gain = []
- offset = []
- badpixels = []
- gain_error = []
- offset_error = []
- gain=np.array(NUC_DL[1][0])
- offset=np.array(NUC_DL[1][1])
- badpixels=np.array(NUC_DL[1][3],dtype=np.ubyte)
- # check quality of NUC elements
- badpixels[np.isnan(gain)] = 1
- badpixels[np.isnan(offset)] = 1
- gain[np.isnan(gain)]=0
- offset[np.isnan(offset)]=0
- badpixels[offset<-1e100] = 1
- offset[offset<-1e100]=0
- # catch cases of error frames being the value "0"
- if NUC_DL[1][4] is 0:
- gain_error = np.zeros(np.shape(gain), dtype=np.int8)
- else:
- gain_error=np.array(NUC_DL[1][4])
- badpixels[np.isnan(gain_error)] = 1
- gain_error[np.isnan(gain_error)]=np.nanmax(gain_error)+1
- if NUC_DL[1][5] is 0:
- offset_error = np.zeros(np.shape(offset), dtype=np.int8)
- else:
- offset_error=np.array(NUC_DL[1][5])
- badpixels[np.isnan(offset_error)] = 1
- offset_error[np.isnan(offset_error)]=np.nanmax(offset_error)+1
- # catch special case for AEF11 and AEF21 in OP1.2a
- if ((port == 11) or (port == 21)) and OP=="OP1.2a":
- exist,time,frames=download_raw_images_by_times(port,t1,int(t1+0.02*1e9),version,verbose=verbose-1)
- if exist:
- # frames=[im.swapaxes(0,1) for im in frames]#somehow the archive gives it now already swapped back (09.07.2018)
- bim=apply_NUC([frames[0]],gain,offset)
- background=get_average_background_recangle(port,bim[0])
- else:
- raise Warning("get_calib_data: cannot find the first frame of the discharge, reconstruction failed")
- return False,0,LUT,0,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
+ else:
+ raise Warning("get_calib_data: no exposure time found")
+ return False,0,0,0,0,0,0,0,cfilter,0,0
+
+ # get Look-Up-Table (LUT)
+ time=int(TimeToNs([2017,9,26],[8,0,0,0]))
+ LUT_DL=download_LUT(port,time,t_exp,emissivity,cfilter,version,testmode=testmode,verbose=verbose-1)
+ if LUT_DL[0]:
+ LUT=LUT_DL[1]
+ del LUT_DL
+ else:
+ raise Warning("get_calib_data: no LUT found")
+ return False,0,0,0,0,0,0,t_exp,cfilter,0,0
+
+ # treat cameras with and w/o filters differently
+ if cfilter==0 and not Camera.split("_")[0]=="Infratec":
+ # --- Infratec Camera case ---
+ if verbose>0:
+ print('get_calib_data: getting NUC arrays')
+ NUC_DL=get_NUC_by_times(port,t0,t1,t_exp,version,testmode,verbose=verbose-1)#download_NUC_by_times(port,t0,t1,t_exp,version)
+ if NUC_DL[0]==False:
+ raise Warning("get_calib_data: NUC was not found")
+ return False,0,LUT,0,0,0,0,t_exp,cfilter,0,0
+ # extract NUC elements
+ gain = []
+ offset = []
+ badpixels = []
+ gain_error = []
+ offset_error = []
+ gain=np.array(NUC_DL[1][0])
+ offset=np.array(NUC_DL[1][1])
+ badpixels=np.array(NUC_DL[1][3],dtype=np.ubyte)
+ # check quality of NUC elements
+ badpixels[np.isnan(gain)] = 1
+ badpixels[np.isnan(offset)] = 1
+ gain[np.isnan(gain)]=0
+ offset[np.isnan(offset)]=0
+ badpixels[offset<-1e100] = 1
+ offset[offset<-1e100]=0
+ # catch cases of error frames being the value "0"
+ if NUC_DL[1][4] is 0:
+ gain_error = np.zeros(np.shape(gain), dtype=np.int8)
+ else:
+ gain_error=np.array(NUC_DL[1][4])
+ badpixels[np.isnan(gain_error)] = 1
+ gain_error[np.isnan(gain_error)]=np.nanmax(gain_error)+1
+ if NUC_DL[1][5] is 0:
+ offset_error = np.zeros(np.shape(offset), dtype=np.int8)
+ else:
+ offset_error=np.array(NUC_DL[1][5])
+ badpixels[np.isnan(offset_error)] = 1
+ offset_error[np.isnan(offset_error)]=np.nanmax(offset_error)+1
+ # catch special case for AEF11 and AEF21 in OP1.2a
+ if ((port == 11) or (port == 21)) and OP=="OP1.2a":
+ exist,time,frames=download_raw_images_by_times(port,t1,int(t1+0.02*1e9),version,testmode=testmode,verbose=verbose-1)
+ if exist:
+ # frames=[im.swapaxes(0,1) for im in frames]#somehow the archive gives it now already swapped back (09.07.2018)
+ bim=apply_NUC([frames[0]],gain,offset)
+ background=get_average_background_recangle(port,bim[0])
else:
- gain=0
- offset=0
- badpixels=0
- gain_error=0
- offset_error=0
- if OP=="OP1.2a":
- back_DL=download_background_by_times(port,t0,t1,t_exp,cfilter,version)
- elif OP=="OP1.2b":
- back_DL=get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version,verbose=verbose-1)
+ raise Warning("get_calib_data: cannot find the first frame of the discharge, reconstruction failed")
+ return False,0,LUT,0,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
+ else:
+ # --- Ircam Camera case ---
+ gain=0
+ offset=0
+ badpixels=0
+ gain_error=0
+ offset_error=0
+
+ # get background
+ if OP=="OP1.2a":
+ back_DL=download_background_by_times(port,t0,t1,t_exp,cfilter,version,testmode=testmode)
+ elif OP=="OP1.2b":
+ back_DL=get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version,testmode,verbose=verbose-1)
+ else:
+ raise Exception("get_calib_data: unknown Operation Phase or calibration method not implemented for this campaign")
+ if not back_DL[0]:
+ raise Warning("get_calib_data: no background image found")
+ return False,0,LUT,0,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
+ background=back_DL[2]
+
+ # apply different versions of temperature calibration
+ if T_version==1:
+ if verbose>0:
+ print('get_calib_data: use temperature calibration version 1')
+ backtime=back_DL[1]
+ backtime=backtime.tolist()
+ divertorpart="all"
+ TCT_dl=get_average_divertor_TC_for_camtemp(port,int(backtime-2e9),int(backtime+2e9),divertorpart)
+ if TCT_dl[0]:
+ refT=np.average(TCT_dl[2])
+ else:
+ TCT_dl=get_average_divertor_TC_for_camtemp(port,int(backtime-6e9),int(backtime+6e9),divertorpart)
+ if TCT_dl[0]:
+ refT=np.average(TCT_dl[2])
else:
- raise Exception("get_calib_data: unknown Operation Phase or calibration method not implemented for this campaign")
- if back_DL[0]:
- background=back_DL[2]
- if T_version==1:
- if verbose>0:
- print('get_calib_data: use temperature calibration version 1')
- backtime=back_DL[1]
- backtime=backtime.tolist()
- divertorpart="all"
- TCT_dl=get_average_divertor_TC_for_camtemp(port,int(backtime-2e9),int(backtime+2e9),divertorpart)
- if TCT_dl[0]:
- refT=np.average(TCT_dl[2])
- else:
- TCT_dl=get_average_divertor_TC_for_camtemp(port,int(backtime-6e9),int(backtime+6e9),divertorpart)
- if TCT_dl[0]:
- refT=np.average(TCT_dl[2])
- else:
- raise Exception("get_calib_data: Unable to find thermocouple data")
- elif T_version==2:
- if verbose>0:
- print('get_calib_data: use temperature calibration version 2')
- frame=background.copy()
- background=get_average_background_recangle(port,background)
- gotit,waterin,waterout=get_cooling_water_temperature_by_time(t1)
- if gotit:
- refT=waterout
- else:
- refT=28.5
- elif T_version==3:
- if verbose>0:
- print('get_calib_data: use temperature calibration version 3, TEST CASE!!!')
- frame=background.copy()
- background=get_average_background_recangle(port,background)
- refT=28.5
- # read LUT V3 (compensated BG signal and photons flux of BG)
- LUT_DL_wall = read_LUT_from_file(port, time, t_exp, 0.45, cfilter, verbose=verbose-1)
- if LUT_DL_wall[0]:
- if verbose>0:
- print('get_calib_data: got the wall LUT')
- LUT_wall = LUT_DL_wall[1]
- index=IR_tools.find_nearest(LUT_wall[1],refT)
- background=background-LUT_wall[0][index]
- refT=0
- del LUT_DL_wall,LUT_wall
- LUT_DL = read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
- if LUT_DL[0]:
- if verbose>0:
- print('get_calib_data: got the V3 LUT')
- LUT = LUT_DL[1]
- del LUT_DL
+ raise Exception("get_calib_data: Unable to find thermocouple data")
+ elif T_version==2:
+ if verbose>0:
+ print('get_calib_data: use temperature calibration version 2')
+ frame=background.copy()
+ background=get_average_background_recangle(port,background)
+ gotit,waterin,waterout=get_cooling_water_temperature_by_time(t1)
+ if gotit:
+ refT=waterout
+ else:
+ refT=28.5
+ elif T_version==3:
+ if verbose>0:
+ print('get_calib_data: use temperature calibration version 3, TEST CASE!!!')
+ frame=background.copy()
+ background=get_average_background_recangle(port,background)
+ refT=28.5
+ # read LUT V3 (compensated BG signal and photons flux of BG)
+ LUT_DL_wall = read_LUT_from_file(port, time, t_exp, 0.45, cfilter, verbose=verbose-1)
+ if LUT_DL_wall[0]:
+ if verbose>0:
+ print('get_calib_data: got the wall LUT')
+ LUT_wall = LUT_DL_wall[1]
+ index=IR_tools.find_nearest(LUT_wall[1],refT)
+ background=background-LUT_wall[0][index]
+ refT=0
+ del LUT_DL_wall,LUT_wall
+ LUT_DL = read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
+ if LUT_DL[0]:
+ if verbose>0:
+ print('get_calib_data: got the V3 LUT')
+ LUT = LUT_DL[1]
+ del LUT_DL
# back_off=estimate_offset(port,program)
# background=(background-back_off)/(back_emissivity)+back_off
- else:
- raise Warning("get_calib_data: Unknown temperature calibration method")
- return False,background,LUT,0,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
-
- else:
- raise Warning("get_calib_data: no background image found")
- return False,0,LUT,0,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
- else:
- raise Warning("get_calib_data: no LUT found")
- return False,0,0,0,0,0,0,t_exp,cfilter,0,0
else:
- raise Warning("get_calib_data: no exposure time found")
- return False,0,0,0,0,0,0,0,cfilter,0,0
+ raise Warning("get_calib_data: Unknown temperature calibration method")
+ return False,background,LUT,0,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
+
+ # get bad pixels
if np.max(badpixels)==0:
if Camera.split("_")[0]=="Infratec":
badpixels=find_badpixels(port,frame-background,offset,niterations=10,tolerance=10,plot_it=False,verbose=verbose-1)#find_badpixels(port,gain,offset)
@@ -2491,10 +2627,9 @@ def get_calib_data(port,program=None,t0=None,t1=None,t6=None,emissivity=0.8,
badpixels = find_badpixels(port, gain, offset, init_bp_list=init_bp_list, niterations=10, tolerance=10,verbose=verbose-1)
if verbose>0:
print('get_calib_data: NUC arrays ready')
+
return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
-# except Exception as E:
-# print("Warning: Program was not found: ",E)
-# return 0,0,0,0,0,0,0,0,0,0
+
def find_badpixels(port, gain, offset, init_bp_list=None, niterations=3,
@@ -2663,33 +2798,37 @@ def download_heatflux_by_program(port,program,time_window=0,threads=1,testmode=F
verbose: interger, level of outputs
returns downlad_heatflux_by_times
"""
- prog=get_program_from_PID(program)
- if prog[0]:
- t1=prog[1]['trigger']['1'][0]
- t6=prog[1]['trigger']['6'][0]
- if time_window==0:
- tstart=t1
- tend=t6
- elif type(time_window)==list or type(time_window)==np.ndarray:
- if len(time_window)>1:
- tstart=int(t1+time_window[0]*1e9)
- tend=int(t1+time_window[1]*1e9)
- else:
- tstart=int(t1)
- tend=int(t1+time_window[0]*1e9)
- elif type(time_window)==int or type(time_window)==float:
- tstart=int(t1)
- tend=int(t1+time_window*1e9)
- else:
- raise Exception("download_heatflux_by_program: given time_window type is not supported")
- if givealpha:
- tstart=tstart-1.1e9
- return download_heatflux_by_times(port=port,tstart=tstart,tend=tend,time_window=0,
- testmode=testmode,version=version,verbose=verbose,
- request=request,moffset=moffset)
- else:
+# prog=get_program_from_PID(program)
+# if prog[0]:
+# t1=prog[1]['trigger']['1'][0]
+# t6=prog[1]['trigger']['6'][0]
+ exist, _, t1, t6 = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if not exist:
print("download_heatflux_by_program: Error! program not found")
return False,0,-1
+
+ if time_window==0:
+ tstart=t1
+ tend=t6
+ elif type(time_window)==list or type(time_window)==np.ndarray:
+ if len(time_window)>1:
+ tstart=int(t1+time_window[0]*1e9)
+ tend=int(t1+time_window[1]*1e9)
+ else:
+ tstart=int(t1)
+ tend=int(t1+time_window[0]*1e9)
+ elif type(time_window)==int or type(time_window)==float:
+ tstart=int(t1)
+ tend=int(t1+time_window*1e9)
+ else:
+ raise Exception("download_heatflux_by_program: given time_window type is not supported")
+ if givealpha:
+ tstart=tstart-1.1e9
+ return download_heatflux_by_times(port=port,tstart=tstart,tend=tend,time_window=0,
+ testmode=testmode,version=version,verbose=verbose,
+ request=request,moffset=moffset)
+
+
def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode=False,
version=0,verbose=0,request=True,moffset=0):
@@ -2704,7 +2843,7 @@ def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode
returns found,time,images
"""
if testmode:
- base="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
+ base=testarchivepath
else:
base=archivepath
OP=IR_tools.get_OP_by_time(time_ns=tstart)
@@ -2734,11 +2873,11 @@ def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode
return False,0,-1
else:## Okay QRT is requested. Maybe there is data available, maybe not, lets find out
if version==0:
- version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"heatflux_DATASTREAM",project=project_ana,Test=testmode,t_from=tstart,t_to=tend)
+ version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"heatflux_DATASTREAM",project=project_ana,testmode=testmode,t_from=tstart,t_to=tend)
larchivepath=base+project_ana+"/"+portpathdict[OP]["AEF"+str(port)]+"heatflux_DATASTREAM/V"+str(version)+"/0/heatflux"
### test for the testsample###
# if version==0:
-# version=get_latest_version("QRT_IRCAM_Test/AEF"+str(port)+"_heatflux_Test_2_DATASTREAM",project=project_ana,Test=testmode)
+# version=get_latest_version("QRT_IRCAM_Test/AEF"+str(port)+"_heatflux_Test_2_DATASTREAM",project=project_ana,testmode=testmode)
# larchivepath=base+project_ana+"/QRT_IRCAM_Test/AEF"+str(port)+"_heatflux_Test_2_DATASTREAM/V"+str(version)+"/0/heatflux"
### end of testsample modifikation###
if verbose>0:
@@ -2773,16 +2912,16 @@ def download_heatflux_mapping_reference(timepoint=None,version=0,testmode=False,
if timepoint==None:
timepoint=int(TimeToNs([2017,9,26],[8,0,0,0]))
if testmode:
- base="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
+ base=testarchivepath
else:
base=archivepath
# OP=IR_tools.get_OP_by_time(time_ns=timepoint)
if version==0:
- version=get_latest_version("QRT_IRCAM/Mapping_reference_DATASTREAM",project=project_ana,Test=testmode,t_from=timepoint)
+ version=get_latest_version("QRT_IRCAM/Mapping_reference_DATASTREAM",project=project_ana,testmode=testmode,t_from=timepoint)
larchivepath=base+project_ana+"/"+"QRT_IRCAM/Mapping_reference_DATASTREAM/V"+str(version)+"/0/reference"
### test for the testsample###
# if version==0:
-# version=get_latest_version("QRT_IRCAM_Test/Mapping_reference_Test_1_PARLOG",project=project_ana,Test=testmode)
+# version=get_latest_version("QRT_IRCAM_Test/Mapping_reference_Test_1_PARLOG",project=project_ana,testmode=testmode)
# larchivepath=base+project_ana+"/QRT_IRCAM_Test/Mapping_reference_Test_1_DATASTREAM/V"+str(version)+"/0/reference"
### end of testsample ###
exist,time,frames=download_images_by_times(larchivepath,starttime=timepoint,stoptime=int(timepoint+1e9),version=version,verbose=verbose-1)
@@ -2897,7 +3036,8 @@ def download_heatflux_mapping_reference(timepoint=None,version=0,testmode=False,
return exist,mappings
-def download_heatflux_scene_model_reference(port,timepoint=None,program=None,version=0,testmode=False,verbose=0):
+def download_heatflux_scene_model_reference(port,timepoint=None,program=None,
+ version=0,testmode=False,verbose=0):
"""
return exist(boolean) and dictonary of the mappings informations
"""
@@ -2908,24 +3048,26 @@ def download_heatflux_scene_model_reference(port,timepoint=None,program=None,ver
if timepoint==None and program==None:
timepoint=int(TimeToNs([2017,8,28],[8,0,0,0]))
elif timepoint==None:
- prog=get_program_from_PID(program)
- if prog[0]:
- timepoint=prog[1]['trigger']['1'][0]
- else:
+# prog=get_program_from_PID(program)
+# if prog[0]:
+# timepoint=prog[1]['trigger']['1'][0]
+# else:
+ exist, _, timepoint, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if not exist:
raise Exception("download_heatflux_scene_model_reference: Error! Program "+program+" not found!")
if testmode:
- base="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
+ base=testarchivepath
else:
base=archivepath
# OP=IR_tools.get_OP_by_time(time_ns=timepoint)
if version==0:
- version=get_latest_version("QRT_IRCAM/"+str(portnam)+"_SceneModel_reference_PARLOG",project=project_ana,Test=testmode,t_from=timepoint)
+ version=get_latest_version("QRT_IRCAM/"+str(portnam)+"_SceneModel_reference_PARLOG",project=project_ana,testmode=testmode,t_from=timepoint)
larchivepath=base+project_ana+"/"+"QRT_IRCAM/"+str(portnam)+"_SceneModel_reference_DATASTREAM/V"+str(version)+"/0/scene%20model%20reference"
larchivepath_par=base+project_ana+"/QRT_IRCAM/"+str(portnam)+"_SceneModel_reference_PARLOG/V"+str(version)+"/parms/header/Pixel_X"
### test for the testsample###
#==============================================================================
# if version==0:
-# version=get_latest_version("QRT_IRCAM_Test/"+str(portnam)+"_SceneModel_reference_PARLOG",project=project_ana,Test=testmode)
+# version=get_latest_version("QRT_IRCAM_Test/"+str(portnam)+"_SceneModel_reference_PARLOG",project=project_ana,testmode=testmode)
# larchivepath=base+project_ana+"/QRT_IRCAM_Test/"+str(portnam)+"_SceneModel_reference_DATASTREAM/V"+str(version)+"/0/scene%20model%20reference"
# larchivepath_par=base+project_ana+"/QRT_IRCAM_Test/"+str(portnam)+"_SceneModel_reference_PARLOG/V"+str(version)+"/parms/header/Pixel_X"
#==============================================================================
@@ -3058,9 +3200,11 @@ def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstar
return False,0,0,0
### lets find out which timepoint or timeintervall is requested, to know the OP, only needed for OP2 changes, but who knows
if program!=None:
- prog=get_program_from_PID(program)
- if prog[0]:
- t1=prog[1]['trigger']['1'][0]
+# prog=get_program_from_PID(program)
+# if prog[0]:
+# t1=prog[1]['trigger']['1'][0]
+ exist, _, t1, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
tstart=t1+int((timepoint-0.02)*1e9)
tend=t1+int((timepoint+0.02)*1e9)
else:
@@ -3070,10 +3214,12 @@ def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstar
else:
t1=tstart
if tend==None:
- proggi=AKF_2.get_program_id(t1)
- prog=get_program_from_PID(proggi)
- if prog[0]:
- tref=prog[1]['trigger']['6'][0]
+ program=AKF_2.get_program_id(t1)
+# prog=get_program_from_PID(program)
+# if prog[0]:
+# tref=prog[1]['trigger']['6'][0]
+ exist, _, _, tref = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if exist:
tend=tref+int((timepoint+0.02)*1e9)
OP=IR_tools.get_OP_by_time(time_ns=t1)
### is the port given?
@@ -3099,7 +3245,7 @@ def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstar
raise Exception("get_heatflux_profile: Unknown port, FLIR data is unsupported at the moment")
else:
### if the program goes up to here, time okay, finger okay,port okay, lets download it
- exist,timo,frames=download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode=False,version=version,verbose=verbose-1)
+ exist,timo,frames=download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode=testmode,version=version,verbose=verbose-1)
if exist:
return extract_heatflux_profile_from_DL(time=(np.asarray(timo)-t1)/1e9,images=frames,profile=int(fingerID%100),finger=int(fingerID/100),time_window=timepoint,inputchecked=inputcheck,verbose=verbose-1)
else:
@@ -3226,7 +3372,8 @@ def extract_temperature_profile_from_DL(port,time,images,profile,finger=None,tim
return False,0,0,0
-def download_divertor_load(port,targetmodule=None,program=None,tstart=None,tend=None,version=0,verbose=0,testmode=False,request=True):
+def download_divertor_load(port,targetmodule=None,program=None,tstart=None,tend=None,
+ version=0,verbose=0,testmode=False,request=True):
"""
"""
TMchadict={
@@ -3280,30 +3427,36 @@ def download_divertor_load(port,targetmodule=None,program=None,tstart=None,tend=
print("input for program and time invalid!")
return False,0,-1,-1
elif program!=None:
- prog=get_program_from_PID(program)
- if prog[0]:
- tstart=prog[1]['trigger']['1'][0]
- tend=prog[1]['trigger']['6'][0]
- else:
- if verbose>0: print("program not found")
+# prog=get_program_from_PID(program)
+# if prog[0]:
+# tstart=prog[1]['trigger']['1'][0]
+# tend=prog[1]['trigger']['6'][0]
+# else:
+ exist, _, tstart, tend = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if not exist:
+ print("download_divertor_load: Error! program not found")
return False,0,-1,-1
else:
if tend==None:
- proggi=AKF_2.get_program_id(tstart)
- prog=get_program_from_PID(proggi)
- if prog[0]:
- tend=prog[1]['trigger']['6'][0]
+ program=AKF_2.get_program_id(tstart)
+# prog=get_program_from_PID(program)
+# if prog[0]:
+# tend=prog[1]['trigger']['6'][0]
+ exist, _, _, tend = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if not exist:
+ print("download_divertor_load: Error! end trigger not found")
+ return False,0,-1,-1
#channels known, time known, okay lets get the data
# tstart=int(tstart-100e6)
OP=IR_tools.get_OP_by_time(time_ns=tstart)
if testmode:
- base="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
+ base=testarchivepath
else:
base=archivepath
if type(port)==int:
port="AEF"+str(port)
if version==0:
- version=get_latest_version(portpathdict[OP][str(port)]+"loads_DATASTREAM",project=project_ana,t_from=tstart,t_to=tend,Test=testmode)
+ version=get_latest_version(portpathdict[OP][str(port)]+"loads_DATASTREAM",project=project_ana,t_from=tstart,t_to=tend,testmode=testmode)
larchivepath=base+project_ana+"/"+portpathdict[OP][str(port)]+"loads_DATASTREAM/V"+str(version)
if type(datcha)==list:#case of several requested channels
load=[]
@@ -3354,7 +3507,7 @@ def download_divertor_load(port,targetmodule=None,program=None,tstart=None,tend=
return exist,time,load,error
-def download_scene_model(port,program=None,timepoint=None,version=0,verbose=0):
+def download_scene_model(port,program=None,timepoint=None,version=0,testmode=False,verbose=0):
"""
"""
if type(port)==int or type(port)==float:
@@ -3365,14 +3518,19 @@ def download_scene_model(port,program=None,timepoint=None,version=0,verbose=0):
dati=datetime.datetime.now()
timepoint=int(TimeToNs([dati.year,dati.month,dati.day],[dati.hour,dati.minute,dati.second,dati.microsecond]))
elif timepoint==None:
- prog=get_program_from_PID(program)
- if prog[0]:
- timepoint=prog[1]['trigger']['1'][0]
- else:
+# prog=get_program_from_PID(program)
+# if prog[0]:
+# timepoint=prog[1]['trigger']['1'][0]
+# else:
+ exist, _, timepoint, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
+ if not exist:
raise Exception("download_scene_model: Error! Program "+program+" not found!")
- base=archivepath
+ if testmode:
+ base=testarchivepath
+ else:
+ base=archivepath
if version==0:
- version=get_latest_version("QRT_IRCAM/"+str(portnam)+"_scene_model_PARLOG",project=project,Test=False,t_from=timepoint)
+ version=get_latest_version("QRT_IRCAM/"+str(portnam)+"_scene_model_PARLOG",project=project,testmode=False,t_from=timepoint)
larchivepath=base+project+"/"+"QRT_IRCAM/"+str(portnam)+"_scene_model_DATASTREAM/V"+str(version)+"/0/"+str(portnam)+"_scene_model"
larchivepath_par=base+project+"/QRT_IRCAM/"+str(portnam)+"_scene_model_PARLOG/V"+str(version)
@@ -3382,7 +3540,7 @@ def download_scene_model(port,program=None,timepoint=None,version=0,verbose=0):
res.close()
goon=True
except urllib.error.URLError as e:
- print('download_heatflux_scene_model_reference: Error! ',e)
+ print('download_scene_model: Error! ',e)
goon=False
if goon:
timepoint=signal_list['values'][0]['meta-data']['timestamp']
@@ -3517,7 +3675,6 @@ def get_program_from_PID(instring):
program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from='
# write the url and load the data
jsonurl = program_url+np.str(w7xstamp0)+'&upto='+np.str(w7xstamp1)
-
try:
res = urllib.request.urlopen(jsonurl)
prog_list = json.loads(res.read().decode('utf-8'))
@@ -3546,76 +3703,272 @@ def get_program_from_PID(instring):
end of the copied functions
@author: thomsen
"""
+
+def get_trigger_from_PID(program, port, testmode=False, verbose=0):
+ '''Gets the nanosecond timestamps of beginning of preparation phase (t0),
+ the start of experiment phase (t1) and the end of the experiment phase (t6).
+ If there was no program in the Archive (calibration or test shot), the
+ function will scan the raw datastream and try to find the program by counting
+ segments of data (with NUC, background, experiment data).
+
+ INPUT
+ -----
+ program: str
+ program string of format YYYYMMDD.NNN (Year, Month, Day,
+ program Number), e.g. '20171026.007'
+ port: int
+ port number of IR camera (e.g. 21 for AEF21 port)
+ testmode: bool, optional
+ switch of whether to search in the Test archive or not
+ (OPTIONAL: default is to search the real Archive)
+ verbose: integer, optional
+ feedback level (details of print messages)
+ (OPTIONAL: if not provided, only ERROR output)
+ RESULT
+ ------
+ exist: bool
+ flag of whether the script was successful
+ t_preparation: int
+ nanosecond time stamp of beginning of preparation phase (t0)
+ t_start: int
+ nanosecond time stamp of beginning of experiment phase (t1)
+ t_end: int
+ nanosecond time stamp of end of experiment phase (t6)
+ '''
+ # handle different formats of port
+ if isinstance(port, str):
+ try:
+ # case port=='31'
+ port = int(port)
+ except:
+ # case port=='... AEF31 ...'
+ port = int(port[port.find('AEF')+3:port.find('AEF')+5])
+ # check if program was defined in archive
+ prog = get_program_from_PID(program)
+ if prog[0]:
+ t_preparation = prog[1]['trigger']['0'][0]
+ t_start = prog[1]['trigger']['1'][0]
+ t_end = prog[1]['trigger']['6'][0]
+
+ return True, t_preparation, t_start, t_end
+ else:
+ if verbose>0:
+ print('get_trigger_from_PID: no program found in Archive. checking datastream...')
+ date_str = program.split('.')[0]
+ prog_no = int(program.split('.')[1])
+ OP = IR_tools.get_OP_by_time(program_str=program)
+ # prepare Datastream url
+ if testmode:
+ stream_name = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+'raw_DATASTREAM'
+ else:
+ stream_name = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+'raw_DATASTREAM'
+ time_from = AKF_2.to_timestamp(date_str+' 00:00:00', fmt=u"%Y%m%d %H:%M:%S")
+ time_to = AKF_2.to_timestamp(date_str+' 23:59:59', fmt=u"%Y%m%d %H:%M:%S")
+ version = AKF_2.get_last_version(stream_name, time_from, time_to)
+ signal_name = stream_name+'/'+version+'/0/raw'
+ # load all time intervals with data in this datastream
+ intervals = AKF_2.get_time_intervals(signal_name, time_from, time_to)
+ frame_start = np.array(intervals[::-1,0])
+ frame_end = np.array(intervals[::-1,1])
+ # identify segments of frames by gaps in time steps
+ dt = abs(intervals[:-1,0]-intervals[1:,0])
+ dt_normal = np.median(dt)
+ i_segment = np.hstack([np.array([0]), np.where(dt>2.5*dt_normal)[0]+1, np.array([len(intervals)])])
+ if verbose>1:
+ print('get_trigger_from_PID: found {0} data segments starting at {1}ns'.format(len(i_segment)-1, frame_start[i_segment[:-1]]))
+ # try to group segments according to normal order (NUC frames, background frames, data frames)
+ if (len(i_segment)-1)%3 == 0:
+ t_programs = []
+ for i in range(len(i_segment)//3):
+ # three data segments:
+ # i*3: NUC, i*3+1: background, i*3+2: experiment phase
+ # --> triggers:
+ # beginning of NUC, beginning of experiment, end of experiment (=one frame before start of next segment)
+ t_programs.append( [frame_start[i_segment[i*3]],
+ frame_start[i_segment[i*3+2]],
+ frame_end[i_segment[(i+1)*3]-1] ] )
+ if verbose>1:
+ print('get_trigger_from_PID: found {0} shots (NUC+BG+data)'.format(len(t_programs)))
+ if len(t_programs)>=prog_no and prog_no>0:
+ t_preparation = t_programs[prog_no-1][0]
+ t_start = t_programs[prog_no-1][1]
+ t_end = t_programs[prog_no-1][2]
+
+ return True, t_preparation, t_start, t_end
+ else:
+ print('get_trigger_from_PID: ERROR! program {0} not found among {1} shots (NUC+BG+data)'.format(prog_no, len(t_programs)))
+
+ return False, 0, 0, 0
+ else:
+ print('get_trigger_from_PID: ERROR! number of segments does not indicate multiple shots.')
+
+ return False, 0, 0, 0
+
+
#%% dummy run of the script
if __name__=='__main__':
print("local function calling")
#%% temperature download and plotting example
- port=31#"AEF50"#"AEF51"'20171114.053'#
- prog="20180911.008"#"20181011.033"#'20171122.035'#'20181011.010'#'20180823.037'#'20170927.020'#"20181011.036"#"20181016.037"#"20180920.042"#"20171109.021"#"20181010.036"
- status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_window=[5,5.1],emi=0.80,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
-# status2,time2,images2=download_raw_images_by_program(port,prog,time_window=0.02,verbose=5)
-# bla=get_calib_data(50,program=prog,verbose=5)
-# success,t,s,profile=extract_temperature_profile_from_DL(port,np.asarray(time-time[0])/1e9,images,profile="TM3h_5_5",verbose=10,reference_time=time[0])
- if status:
- plt.figure()
- plt.imshow(images[-1],vmin=330,vmax=1000,cmap=exJet)
- cb=plt.colorbar()
- cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20)
- cb.ax.tick_params(labelsize=20)
- plt.title("AEF{0}".format(port))
+# port=31#"AEF50"#"AEF51"'20171114.053'#
+# prog="20180911.008"#"20181011.033"#'20171122.035'#'20181011.010'#'20180823.037'#'20170927.020'#"20181011.036"#"20181016.037"#"20180920.042"#"20171109.021"#"20181010.036"
+# status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_window=[5,5.1],emi=0.80,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
+## status2,time2,images2=download_raw_images_by_program(port,prog,time_window=0.02,verbose=5)
+## bla=get_calib_data(50,program=prog,verbose=5)
+## success,t,s,profile=extract_temperature_profile_from_DL(port,np.asarray(time-time[0])/1e9,images,profile="TM3h_5_5",verbose=10,reference_time=time[0])
+# if status:
+# plt.figure()
+# plt.imshow(images[-1],vmin=330,vmax=1000,cmap=exJet)
+# cb=plt.colorbar()
+# cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20)
+# cb.ax.tick_params(labelsize=20)
+# plt.title("AEF{0}".format(port))
+
#%% heatflux test
-# port=10
-# prog="20180904.015"
-#
-# for prog in ["20171207.016"]:#"20180821.012"]:#"20180823.016","20180918.045","20181016.037","20181009.024","20181016.016"]:#"20171207.024","20180814.024","20181016.016","20181010.036"]:
-# for port in [10,11,20,21,30,31,40,41,50,51]:
-# status,time,images=download_heatflux_by_program(port,prog,time_window=0.9,threads=4,version=2,verbose=3,givealpha=False,request=True)
-## status,time,load,error=download_divertor_load(port,targetmodule=['all','tm1h'],program=prog,verbose=5,version=3)
-# if status:
-# print(prog,port,"OK")
-# else:
-# print(prog,port,"missing")
-# plt.figure(),plt.imshow(images[0],vmin=0)
-# plt.figure(),plt.imshow(images[50],vmin=0)
+# port = 20
+# prog = "20181016.016"
+#
# status,mapping=download_heatflux_mapping_reference(verbose=4)
-# test=get_heatflux_profile(20,1605,timepoint=1,program="20171109.008",verbose=4)
+#
+# plt.figure()
+# plt.subplot(2,1,1)
+# t_start = 2.0
+# status,times1,images1=download_heatflux_by_program(port,prog,time_window=[t_start,t_start+0.1],threads=4,version=2,verbose=3,givealpha=False,request=False)
+# plot_frame1 = np.nan_to_num(images1[0])/1E6
+# plot_frame1[plot_frame1<0.1] = 0
+# q_max = np.nanmax(plot_frame1)
+# q_min = np.nanpercentile(plot_frame1[plot_frame1>0], 1)
+# plt.scatter(mapping['X'], mapping['Y'], s=1, c=plot_frame1, cmap='jet', vmin=q_min)
+#
+# plt.colorbar(label='q [MW/m²]')
+# plt.title('t-t_0={0:.1f}s'.format(t_start))
+#
+# plt.subplot(2,1,2)
+# t_start = 10.0
+# status,times2,images2=download_heatflux_by_program(port,prog,time_window=[t_start,t_start+0.1],threads=4,version=2,verbose=3,givealpha=False,request=False)
+# plot_frame2 = np.nan_to_num(images2[0])/1E6
+# plot_frame2[plot_frame2<0.1] = 0
+# plt.scatter(mapping['X'], mapping['Y'], s=1, c=plot_frame2, cmap='jet', vmin=q_min, vmax=q_max)
+# plt.colorbar(label='q [MW/m²]')
+# plt.title('t-t_0={0:.1f}s'.format(t_start))
+# plt.suptitle('Heat flux in module {0} program {1}'.format(port, prog))
+# plt.tight_layout(rect=[0, 0, 1, 0.95])
+# plt.show()
+
+#%% heat flux profile test
+# port = 20
+# prog = "20171109.008"
+# finger_no = 1605
+#
+# status, profile_time, s, profiles=get_heatflux_profile(port,finger_no,timepoint=1,program=prog,verbose=4)
+# plt.figure()
+# plt.plot(s, profiles[0]/1E6)
+# plt.xlabel('s [m]')
+# plt.ylabel('q [MW/m²]')
+# plt.title('heat flux along finger {0}\nin module {1} @ t-t1={2:.2f}s'.format(finger_no, port, profile_time[0]))
+# plt.show()
#%% loads test
-# port=10
-# prog="20171207.016"#"20181011.015"#"20171114.052"
-# for port in [10,11,20,21,30,31,40,41,50,51]:
-# status,time,load,error=download_divertor_load(port,targetmodule=['all','tm1h'],program=prog,verbose=5)
+# prog = "20181016.016"
+# loads = []
+# times = []
+# labels = []
+# for port in [11,20,21,30,31,40,41]:
+# print('loading loads of port {0} in {1}'.format(port, prog))
+# exist,time,load,error=download_divertor_load(port,targetmodule=['all'],program=prog,request=False,verbose=5)
+# if exist:
+# load_start = np.mean(load[0][:10])
+# load_end = np.mean(load[0][-10:])
+# load_offset = np.linspace(load_start, load_end, len(time))
+# load_offset = 0
+# times.append(time)
+# loads.append(load[0]-load_offset)
+# labels.append('module {0}'.format(port))
+#
+# plt.figure()
+# for i in range(len(times)):
+# plt.plot((times[i] - times[i][0])/1E9, loads[i]/1E3, label=labels[i], linewidth=1, linestyle='--')
+# plt.legend()
+# plt.title('Integral heat load - program {0}'.format(prog))
+# plt.xlabel('t - t_0 [s]')
+# plt.ylabel('P_divertor [kW]')
+# plt.tight_layout()
+# plt.show()
+
#%% scene model test
# port=10
# exist,model=download_scene_model(port)
+# keys = ['CAD', 'FOV', 'PFC', 'distance', 'angle', 'emissivity', 'x', 'z', 'phi']
+# if exist:
+# plt.figure()
+# for i in range(len(keys)):
+# plt.subplot(3,3,i+1)
+# plt.imshow(model[keys[i]])
+# plt.colorbar()
+# plt.title(keys[i])
+# plt.tight_layout()
+# plt.show()
+
#%% HDF5 writing test
-# bla=get_temp_from_raw_by_program_fullthreads(51,prog,time_window=[0,4],threads=4)
-# bla2=get_nuced_raw_by_program_fullthreads(51,prog,time_window=[0,4],threads=4)
-# port=41
-# program="20180925.013"
-# prog=get_program_from_PID(program)
+# import h5py as h5
+## bla=get_temp_from_raw_by_program_fullthreads(51,prog,time_window=[0,4],threads=4)
+## bla2=get_nuced_raw_by_program_fullthreads(51,prog,time_window=[0,4],threads=4)
+# port = 10
+# program = "20180925.013"
+# prog = get_program_from_PID(program)
# if prog[0]:
-# t0=prog[1]['trigger']['0'][0]
-# t1=prog[1]['trigger']['1'][0]
-# t6=prog[1]['trigger']['6'][0]
-# for port in [10]:
-## bla=download_raw_images_by_times(port,t0,t1,verbose=5)
-# bla=get_temp_from_raw_by_program(port,program,time_window=[0,4],emi=0.82,T_version=1,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=True,verbose=5)
-# import h5py as h5
-# File=h5.File("AEF"+str(port)+"_"+program+"_temp_v1_b.h5",'w')
-# imags=np.asarray(bla[2]).swapaxes(0,2).swapaxes(0,1)
-# shapi=np.shape(imags)
-# dset=File.create_dataset('images',shape=(shapi[0],shapi[1],shapi[2]),dtype='uint16',chunks=(shapi[0],shapi[1],1))
-# imags=np.ascontiguousarray(imags)
-# dset.write_direct(imags)
-# File.create_dataset('timestamps', data=list(bla[1]), dtype='uint64')#,compression="gzip")
-# File.close()
-
-#%% get hot cold image test
- port=10
- program="20180904.002"
+# t0 = prog[1]['trigger']['0'][0]
+# t1 = prog[1]['trigger']['1'][0]
+# t6 = prog[1]['trigger']['6'][0]
+#
+## bla=download_raw_images_by_times(port,t0,t1,verbose=5)
+# bla = get_temp_from_raw_by_program(port,program,time_window=[0,4],emi=0.82,
+# T_version=1,version=0,threads=4,give_ERROR=False,
+# use_firstframe_as_background=True,verbose=5)
+#
+# filename = "AEF"+str(port)+"_"+program+"_temp_v1_b.h5"
+# File = h5.File(filename,'w')
+# frames = np.asarray(bla[2]).swapaxes(0,2).swapaxes(0,1)
+# n_rows, n_cols, n_frames = np.shape(frames)
+# dset = File.create_dataset('images',shape=(n_rows,n_cols,n_frames),dtype='uint16',chunks=(n_rows,n_cols,1))
+# imags = np.ascontiguousarray(frames)
+# dset.write_direct(imags)
+# File.create_dataset('timestamps', data=list(bla[1]), dtype='uint64')#,compression="gzip")
+# File.close()
+#
+# # test reading
+# File = h5.File(filename,'r')
+# times = np.array(File['timestamps'])
+# frames2 = File['images']
+# n_frames2 = frames2.shape[2]
+# if n_frames == n_frames2:
+# i_frames = np.random.randint(low=0, high=n_frames-1 ,size=3)
+# for i in range(3):
+# print('testing frame', i_frames[i])
+# if (np.nan_to_num(frames[:,:,i_frames[i]]) == frames2[:,:,i_frames[i]]).all():
+# print(' original and h5 file verions identical!')
+# else:
+# print(' total difference of original and h5 file:', np.sum(np.nan_to_num(frames[:,:,i_frames[i]]) - frames2[:,:,i_frames[i]]))
+# File.close()
+
+#%% get coldframe from NUC test
+# port = 31
+# program = '20171026.007' #"20190621.001"
+# testmode = False #True
+# exist, coldframe = get_NUCed_coldframe_by_program(port, program, exposure=None,
+# version=0, testmode=testmode,
+# plot_it=True, verbose=3)
+
+#%% get trigger times
+# port = 31
+# program = '20190621.001' #'20190621.001' #'20171026.007'
+# time_window = 0 #0 #0.02
+# testmode = True #True #False
+# verbose = 1
+#
+# exist, t0, t1, t6 = get_trigger_from_PID(program, port, testmode, verbose=1)
+# if exist:
+# print('data of port {0} in program {1}:\n t_preparation {2}ns\n t_start {3}ns\n t_end {4}ns'.format(port, program, t0, t1, t6))
- exist, coldframe = get_NUCed_coldframe_by_program(port, program, exposure=None, version=0, plot_it=True, verbose=3)
\ No newline at end of file