Skip to content

GitLab

Commit b84fad53 authored by Holger Niemann's avatar Holger Niemann
Browse files

Merge branch 'Holgers' into 'master' 

Holgers

See merge request holn/IR_data_access!42
parents 7d8d22e8 452c20fe
Hide whitespace changes
Inline Side-by-side
IR_image_tools.py
View file @ b84fad53
# -*- coding: utf-8 -*-
"""
Created on Wed May 9 14:56:32 2018
Version: 3.3.0
Version: 3.3.2
@author: Holger Niemann, Peter Drewelow, Yu Gao
mainly to clean up the downloadversionIRdata code
......@@ -521,6 +521,33 @@ def check_dublicates_2(array):
seen.add(x)
return uniq,seen
def get_work_list(pipepath,typ="q"):
"""
"""
today=datetime.datetime.now()
cam_programs=[]
if typ=='q' or typ=='load':
f=open(pipepath+str(today.year)+str(today.month)+"_"+typ+"_requests.txt")
else:
reasons=[]
f=open(pipepath+"problematic_programs.txt")
for line in f:
koline=line.split("\t")
if len(koline)>1:
prog=koline[0]
if typ=='q' or typ=='load':
cam_programs.append((prog,koline[1].split("\n")[0]))
else:
cam_programs.append((prog,koline[1]))
reasons.append(koline[2].split("\n")[0])
f.close()
if typ=='q' or typ=='load':
bla=check_dublicates_2(cam_programs)
cam_programs=bla[0]
return cam_programs
else:
return cam_programs,reasons
#%% functions regarding wetted area calculation
def read_finger_info(file_name=None, OP='OP1.2b', verbose=0):
......@@ -630,7 +657,7 @@ def derive_wetted_area_per_module(heat_flux, mapping, mode='average', q_max=None
noise_threshold: float, optional
minimum heat flux level to crop heat_flux to, if heat flux has negative values
(OPTIONAL: default is 200kW/m²)
ports_loaded: list or str or int, optinal
ports_loaded: list or str or int, optional if mode not 'average'
label of divertor modules provided in heat_flux array for plots;
int of port number for single divertor data and list of
port numbers for heat flux from multiple divertor modules; #
......@@ -655,12 +682,23 @@ def derive_wetted_area_per_module(heat_flux, mapping, mode='average', q_max=None
* 3D: heat flux from multiple divertor modules
and the mode to derive q_max ('module', 'average', 'finger'):
* '2D' + 'module' or 'average' --> one value for total_wetted_area and q_max
* '3D' + 'average' --> one value for total_wetted_area and q_max
* '3D' + 'average' --> 1D numpy arrays with two values for total_wetted_area and q_max (upper and lower divertors)
* '3D' + 'module' --> 1D numpy arays with a value for each torus module (first dimension of heat_flux)
* '2D' + 'finger' --> 1D numpy arays with a value for each divertor finger
* '3D' + 'finger' --> 2D numpy arays with a value for each torus module and each divertor finger
'''
#check input
heat_flux_dim = len(np.shape(heat_flux))
if mode == 'average' and ports_loaded == None and heat_flux_dim>2:
raise Exception("derive_wetted_area_per_module: ports must be specified in average mode since V3.3.2")
elif mode == 'average' and heat_flux_dim>2:
try:
llen=len(ports_loaded)
except:
raise Exception("derive_wetted_area_per_module: each divertor need a description to calcualte proper the wetted area!")
else:
if llen!=len(heat_flux):
raise Exception("derive_wetted_area_per_module: number of given divertors and number of descriptions does not match!")
# prepare mapping and finger information
finger_dic = read_finger_info(verbose=verbose-1)
finger_ID = finger_dic['ID']
......@@ -682,14 +720,31 @@ def derive_wetted_area_per_module(heat_flux, mapping, mode='average', q_max=None
if verbose>0:
print('derive_wetted_area_per_module: set heat_flux < {0:.1f}kW/m² to 0'.format(noise_threshold/1E3))
heat_flux_dim = len(np.shape(heat_flux))
# reduce dimension of heat_flux if in 'average' mode
if heat_flux_dim == 2 and mode == 'average':
mode = 'module'
elif heat_flux_dim == 3 and mode == 'average':
heat_flux = np.nanmean(heat_flux, axis=0)
heat_flux_dim = 2
ports_loaded = 'mean heat flux'
# heat_flux = np.nanmean(heat_flux, axis=0)
## sort the divertors
updiv=([])
downdiv=[]
for i in range(len(ports_loaded)):
# entries in the array/list are either int or str or even float
try:
port=int(ports_loaded[i])
except: #okay it is not an int or an int like string
## what can it be? 'AEFXX'? But what about OP2 with the A or K ports? Still be 3 letters
port=int(ports_loaded[i][3:])
if port%10==0:
downdiv.append(heat_flux[i])
else:
updiv.append(heat_flux[i])
heat_flux=np.array([np.nanmean(np.asarray(updiv),axis=0),np.nanmean(np.asarray(downdiv),axis=0)])
del downdiv,updiv
# heat_flux_dim = 3
ports_loaded = ['upper divertor','lower divertor']#'mean heat flux'
mode = 'module'
if verbose>0:
print('derive_wetted_area_per_module: averaged 3D heat flux array over first dimension')
......
downloadversionIRdata.py
View file @ b84fad53
......@@ -4,7 +4,7 @@ Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018
last update on Fr Nov 23 15:37:00 2018
Version: 3.3.0
Version: 3.3.2
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
@author: holn
"""
......@@ -21,7 +21,9 @@ from IR_config_constants import archivepath,testarchivepath,portcamdict,camlimdi
#except:
# import read_restdb as AKF_1
import datetime
import urllib
import urllib.request
import urllib.error
import json
from PIL import Image
from io import BytesIO
......@@ -458,13 +460,15 @@ def download_hot_cold_reference_by_times(port,exposure,starttime=150390720000000
Uses first calibration frames if time is not defined.
"""
OP=IR_tools.get_OP_by_time(time_ns=starttime)
if isinstance(port,int):
port="AEF"+str(port)
if testmode:
larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
larchivepath=testarchivepath+project+"/"+portpathdict[OP][port]+"raw_"
else:
larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
larchivepath=archivepath+project+"/"+portpathdict[OP][port]+"raw_"
# NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if version==0:
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime)
version=get_latest_version(portpathdict[OP][port]+"raw_DATASTREAM",t_from=starttime)
try:
path_string = larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9))
res = urllib.request.urlopen(path_string)
......@@ -506,7 +510,7 @@ def download_hot_cold_reference_by_times(port,exposure,starttime=150390720000000
else:
return False,0,0
def download_background_by_program(port,program,exposure,version=0,testmode=False,verbose=0):
def download_background_by_program(port,program,exposure,camera_filter=0,version=0,testmode=False,verbose=0):
'''
returned exist,time,frame
'''
......@@ -519,7 +523,7 @@ def download_background_by_program(port,program,exposure,version=0,testmode=Fals
# stoptime=prog[1]['trigger']['1'][0]
exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
if exist:
return download_background_by_times(port,starttime,stoptime,exposure,version)
return download_background_by_times(port,starttime,stoptime,exposure,camera_filter=camera_filter,version=version)
else:#except:
print("download_background_by_program: Error! cannot find the program {0}".format(program))
return False,0,0
......@@ -1071,6 +1075,7 @@ def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,ver
# enddate=enddate.isoformat()
else:
starttime=int(t1-100)
tstart=t1
if time_window==0:
stoptime=t6
else:
......@@ -1140,7 +1145,7 @@ def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0,testmode=True):
counter=-1
for version0 in versions:
counter+=1
print(versions)
# print(versions)
try:
if version0>0:
found=False
......@@ -1872,7 +1877,7 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=
verbose=0,testmode=False,framerate='max'):
"""
"""
FLIR=False
FLIR=False
trigger_exist, t0, t1, t6 = get_trigger_from_PID(program, portnr, testmode, verbose=verbose-1)
if not trigger_exist:
raise Exception("get_temp_from_raw_by_program: unknown Program")
......@@ -1881,14 +1886,18 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=
OP=IR_tools.get_OP_by_time(time_ns=t1)
if type(portnr) is str:
try:
camera=portcamdict[OP][portnr]
if camera.split("_")[0]=='FLIR':
FLIR=True
else:
FLIR=False
portnr=int(portnr.split("AEF")[1])
except Exception as E:
raise Exception("get_temp_from_raw_by_program: unknown Port!"+E)
portnr=int(portnr)
FLIR=False
except:
try:
camera=portcamdict[OP][portnr]
if camera.split("_")[0]=='FLIR':
FLIR=True
else:
FLIR=False
portnr=int(portnr.split("AEF")[1])
except Exception as E:
raise Exception("get_temp_from_raw_by_program: unknown Port!"+E)
elif isinstance(portnr, (int, np.integer, float, np.float)):
FLIR=False
portnr=int(portnr)
......@@ -1978,7 +1987,7 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=
else:
# --- IRcam case ---
#
# get calibration data
# get calibration data
cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0=t0,t1=t1,t6=t6,emissivity=emi,T_version=T_version,version=version,back_emissivity=back_emissivity,testmode=testmode,verbose=verbose-1)
if not cexist:
if verbose>0:
......@@ -2691,9 +2700,11 @@ def make_FOV_mask(port):
r1=points[2]
if port==50:
exposure=50
cfilter=1
else:
exposure=9
da,time,back=download_background_by_program(port,"20171109.045",exposure)
cfilter=0
da,time,back=download_background_by_program(port,"20171109.045",exposure,camera_filter=cfilter)
# fig = plt.figure()
# plt.imshow(back,vmin=np.average(back)-200,vmax=np.average(back)+500)
# inner_circle = mlt.patches.Circle((x0,y0), r1,color = 'r', fill = False)
......@@ -2715,7 +2726,7 @@ def make_FOV_mask(port):
else:
plt.imsave(config_path+portcamdict['OP1.2a']['AEF'+str(port)]+"\\"+"AEF"+str(port)+"_FOV.png",FOV)
else:
raise Exception("make_FOV_mask: cannot find the background frame to create the FOV file")
raise Exception("make_FOV_mask: cannot find the background frame to create the FOV file",exposure,port)
def get_FOV_mask(port):
"""
......@@ -2811,12 +2822,15 @@ def download_heatflux_by_program(port,program,time_window=0,threads=1,testmode=F
tstart=t1
tend=t6
elif type(time_window)==list or type(time_window)==np.ndarray:
if len(time_window)>1:
tstart=int(t1+time_window[0]*1e9)
tend=int(t1+time_window[1]*1e9)
else:
tstart=int(t1)
tend=int(t1+time_window[0]*1e9)
try:
if len(time_window)>1:
tstart=int(t1+time_window[0]*1e9)
tend=int(t1+time_window[1]*1e9)
else:
tstart=int(t1)
tend=int(t1+time_window[0]*1e9)
except Exception as E:
raise Exception("download_heatflux_by_program: an Exception occured interpretating the time_window: ",E)
elif type(time_window)==int or type(time_window)==float:
tstart=int(t1)
tend=int(t1+time_window*1e9)
......@@ -2891,17 +2905,26 @@ def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode
if verbose>0:
print(now,"download_heatflux_by_times: heat flux data is not available")
if request:
f=open(heatflux_requestlist_path+str(now.year)+str(now.month+moffset)+"_q_requests.txt",'a')
try:
programid=AKF_2.get_program_id(tstart)
except Exception as E:
if verbose>0:
print('download_heatflux_by_times: Error! ',E)
programid=str(tstart)
f.write(programid+"\tAEF"+str(port)+"\n")
f.close()
if verbose>0:
print("download_heatflux_by_times: heat flux calculation request logged for automatic processing (within ca. 1 day)")
programid=str(tstart)
cam_progs=IR_tools.get_work_list(heatflux_requestlist_path,typ='load')
cam_progs_ig,reasons=IR_tools.get_work_list(heatflux_requestlist_path,typ='ignore')
if (programid,"AEF"+str(port)) not in cam_progs and (programid,"AEF"+str(port)) not in cam_progs_ig:
f=open(heatflux_requestlist_path+str(now.year)+str(now.month+moffset)+"_q_requests.txt",'a')
f.write(programid+"\tAEF"+str(port)+"\n")
f.close()
if verbose>0:
print("download_heatflux_by_times: heat flux calculation request logged for automatic processing (within ca. 1 day)")
elif verbose>0:
if (programid,'AEF'+str(port)) in cam_progs:
print("download_heatflux_by_times: request exist already")
else:
pid=cam_progs_ig.index((programid,'AEF'+str(port)))
print("download_heatflux_by_times: request ignored, data not available, reason: {0}".format(reasons[pid]))
return exist,time,frames
def download_heatflux_mapping_reference(timepoint=None,version=0,testmode=False,
......@@ -3099,7 +3122,7 @@ def download_heatflux_scene_model_reference(port,timepoint=None,program=None,
else:
return False,{}
def give_finger_ID(profile,finger=None):
def give_finger_ID(profile,finger=None,verbose=0):
"""
profile: string, integer or float, string: "TM3h_5_5" or "1605" or "16.05" or "5", similar for float and integer. single number only if the finger is given!
finger: string or integer, string "TM3h_5" or "16" or 16 or 1600 or None
......@@ -3130,17 +3153,20 @@ def give_finger_ID(profile,finger=None):
fingerID=int(profile)
success=True
else:
success=False
success=False
if verbose>0:
print("give_finger_ID: type of profile is not supported!")
else:#cases with both are given
if finger==str:
if type(finger)==str:
finger=finger.lower()
if "tm" in finger:
fing=finger.split("_")
offset=offsets[fing[0]]
finger=offset+int(prof[1])
finger=offset+int(fing[1])
else:
finger=int(finger)
if finger%100==0:
#finger is now an int
if finger%100==0: #check whether the finger format is 1600 or equal to it, reduce it to 16 or equal
finger=int(finger/100)
if type(profile)==int and type(finger)==int:#simple case
if finger<138 and profile<18:
......@@ -3148,26 +3174,39 @@ def give_finger_ID(profile,finger=None):
fingerID=finger*100+profile
else:
success=False
if verbose>0:
print("give_finger_ID: given finger or profile is out of valid range! (finger 0-137, profiles 0-17)")
### combinations left: str-str, float-str, float-int, str-int, int-str
elif type(profile)==str or type(profile)==float:#combinations str-str and str-int
try:### str: by logic only integer should be left if input is a valid one, float: in theory for a valid no further digits should be there
if type(profile)==float:
if profile<1.0:
profile=profile*100
fingerID=finger*100+int(profile)
success=True
except:
success=False
if verbose>0:
print("give_finger_ID: profile and finger defined, profile is not in a valid format!")
elif type(profile)==int:
if profile<18:
fingerID=finger*100+profile
success=True
else:
success=False
if verbose>0:
print("give_finger_ID: profile is out of valid range (0-17)")
else:
success=False
if verbose>0:
print("give_finger_ID: type of profile is not supported!")
if success:#last test
if fingerID%100<18 and fingerID//100<138 and not fingerID==-1:
success=True
else:
success=False
if verbose>0:
print("give_finger_ID: invalid fingerID found. Please check your input.")
return success,fingerID
def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstart=None,
......@@ -3495,16 +3534,26 @@ def download_divertor_load(port,targetmodule=None,program=None,tstart=None,tend=
if verbose>0:
print(now,"download_divertor_load: divertor_load data is not available, creating request")
if request:
f=open(heatflux_requestlist_path+str(now.year)+str(now.month)+"_load_requests.txt",'a')
try:
programid=AKF_2.get_program_id(tstart)
except Exception as E:
if verbose>0:
print('download_divertor_load: Error! ',E)
programid=str(tstart)
f.write(programid+"\t"+str(port)+"\n")
f.close()
cam_progs=IR_tools.get_work_list(heatflux_requestlist_path,typ='load')
cam_progs_ig,reasons=IR_tools.get_work_list(heatflux_requestlist_path,typ='ignore')
if (programid,str(port)) not in cam_progs and (programid,str(port)) not in cam_progs_ig:
f=open(heatflux_requestlist_path+str(now.year)+str(now.month)+"_load_requests.txt",'a')
f.write(programid+"\t"+str(port)+"\n")
f.close()
if verbose>0:
print(now,"download_divertor_load: request created")
elif verbose>0:
if (programid,str(port)) in cam_progs:
print(now,"download_divertor_load: request exist already")
else:
pid=cam_progs_ig.index((programid,str(port)))
print(now,"download_divertor_load: request ignored, data not availabel, reason: {0}".format(reasons[pid]))
return exist,time,load,error
def download_scene_model(port,program=None,timepoint=None,version=0,testmode=False,verbose=0):
......@@ -3740,10 +3789,44 @@ def get_trigger_from_PID(program, port, testmode=False, verbose=0):
try:
# case port=='31'
port = int(port)
port = "AEF"+str(int(port))
except:
# case port=='... AEF31 ...'
port = int(port[port.find('AEF')+3:port.find('AEF')+5])
avports=[]
for key in portpathdict:
avports=avports+list(portpathdict[key].keys())
portprefix=[]
for portstr in avports:
if portstr[0:3] not in portprefix:
portprefix.append(portstr[0:3])
if len(port)<6:
if port not in avports:
print("get_trigger_from_PID: unknown port!",port,avports)
return False,0,0,0
else:
gotit=False
for key in portprefix:
if key in port:
port = int(port[port.find(key)+3:port.find(key)+5])
port=key+str(port)
gotit=True
break
if not gotit:
print("get_trigger_from_PID: unknown port!",port,avports)
return False,0,0,0
# if 'AEF' in port:
# port = int(port[port.find('AEF')+3:port.find('AEF')+5])
else:
try:
port="AEF"+str(int(port))
except:
print("get_trigger_from_PID: port format not supported")
return False,0,0,0
# check if program was defined in archive
if not isinstance(program, str):
program=str(program)
if len(program.split(".")[0])<3:
program=program+"0"
prog = get_program_from_PID(program)
if prog[0]:
t_preparation = prog[1]['trigger']['0'][0]
......@@ -3759,13 +3842,17 @@ def get_trigger_from_PID(program, port, testmode=False, verbose=0):
OP = IR_tools.get_OP_by_time(program_str=program)
# prepare Datastream url
if testmode:
stream_name = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+'raw_DATASTREAM'
stream_name = testarchivepath+project+"/"+portpathdict[OP][port]+'raw_DATASTREAM'
else:
stream_name = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+'raw_DATASTREAM'
stream_name = archivepath+project+"/"+portpathdict[OP][port]+'raw_DATASTREAM'
time_from = AKF_2.to_timestamp(date_str+' 00:00:00', fmt=u"%Y%m%d %H:%M:%S")
time_to = AKF_2.to_timestamp(date_str+' 23:59:59', fmt=u"%Y%m%d %H:%M:%S")
version = AKF_2.get_last_version(stream_name, time_from, time_to)
signal_name = stream_name+'/'+version+'/0/raw'
if version is not None:
signal_name = stream_name+'/'+version+'/0/raw'
else:
print("get_trigger_from_PID: no version with data found!")
return False,0,0,0
# load all time intervals with data in this datastream
intervals = AKF_2.get_time_intervals(signal_name, time_from, time_to)
frame_start = np.array(intervals[::-1,0])
......@@ -3803,8 +3890,8 @@ def get_trigger_from_PID(program, port, testmode=False, verbose=0):
print('get_trigger_from_PID: ERROR! number of segments does not indicate multiple shots.')
return False, 0, 0, 0
#%% dummy run of the script
if __name__=='__main__':
......@@ -3814,9 +3901,9 @@ if __name__=='__main__':
# port=31#"AEF50"#"AEF51"'20171114.053'#
# prog="20180911.008"#"20181011.033"#'20171122.035'#'20181011.010'#'20180823.037'#'20170927.020'#"20181011.036"#"20181016.037"#"20180920.042"#"20171109.021"#"20181010.036"
# status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_window=[5,5.1],emi=0.80,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
## status2,time2,images2=download_raw_images_by_program(port,prog,time_window=0.02,verbose=5)
## bla=get_calib_data(50,program=prog,verbose=5)
## success,t,s,profile=extract_temperature_profile_from_DL(port,np.asarray(time-time[0])/1e9,images,profile="TM3h_5_5",verbose=10,reference_time=time[0])
# status2,time2,images2=download_raw_images_by_program(port,prog,time_window=0.02,verbose=5)
# bla=get_calib_data(50,program=prog,verbose=5)
# success,t,s,profile=extract_temperature_profile_from_DL(port,np.asarray(time-time[0])/1e9,images,profile="TM3h_5_5",verbose=10,reference_time=time[0])
# if status:
# plt.figure()
# plt.imshow(images[-1],vmin=330,vmax=1000,cmap=exJet)
......
setup.py
View file @ b84fad53
......@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name = 'ir-data-access',
version = '3.2.4',
version = '3.3.2',
author = 'Holger Niemann, Peter Drewelow',
author_email = 'holger.niemann@ipp.mpg.de',
description = 'Access Frontend for IR camera data',
......@@ -25,7 +25,9 @@ setup(
'data/AEF31_coldframes_background_fails_real.txt',
'data/AEF40_coldframes_background_fails_real.txt',
'data/AEF41_coldframes_background_fails_real.txt',
'data/AEF51_coldframes_background_fails_real.txt'])
'data/AEF51_coldframes_background_fails_real.txt',
'data/finger_info_HHF.csv',
'data/finger_info_TDU.csv'])
],
install_requires = [
'archivedb>=0.2.0'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment