Commit d92fad3e authored by Holger Niemann's avatar Holger Niemann
Browse files

Merge branch 'Holgers' into 'master'

Holgers

See merge request holn/IR_data_access!31
parents 08ec0353 9282d458
...@@ -4,7 +4,7 @@ Created on Wed Oct 25 15:51:46 2017 ...@@ -4,7 +4,7 @@ Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018 updated on Tue Aug 21 10:20:00 2018
last update on Fr Nov 23 15:37:00 2018 last update on Fr Nov 23 15:37:00 2018
Version: 3.2.1 Version: 3.2.2
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes ) (Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
@author: holn @author: holn
""" """
...@@ -45,6 +45,7 @@ except: ...@@ -45,6 +45,7 @@ except:
def get_latest_version(stream,project="W7X",Test=False,t_from=None,t_to=None,program=None): def get_latest_version(stream,project="W7X",Test=False,t_from=None,t_to=None,program=None):
"""Find out latest version of given stream """Find out latest version of given stream
:param stream url stream of interest :param stream url stream of interest
:return int of version number of None if non-versioned or non-existing stream :return int of version number of None if non-versioned or non-existing stream
...@@ -75,6 +76,10 @@ def get_latest_version(stream,project="W7X",Test=False,t_from=None,t_to=None,pro ...@@ -75,6 +76,10 @@ def get_latest_version(stream,project="W7X",Test=False,t_from=None,t_to=None,pro
except urllib.error.HTTPError as ex: except urllib.error.HTTPError as ex:
msg = ex.read() msg = ex.read()
raise RuntimeError(msg) raise RuntimeError(msg)
except Exception as E:
raise RuntimeError(E)
else:
response.close()
# detect unversioned or non-existing stream # detect unversioned or non-existing stream
if d["versionInfo"] == []: if d["versionInfo"] == []:
return None return None
...@@ -113,34 +118,16 @@ def read_program(timestamp_start,timestamp_end=0,tol=60): ...@@ -113,34 +118,16 @@ def read_program(timestamp_start,timestamp_end=0,tol=60):
print('read_program: Error opening URL') print('read_program: Error opening URL')
print(e) print(e)
return False,0 return False,0
except Exception as e:
print('read_program: Error opening URL',e)
return False,0
else: else:
prog_string=prog_raw.decode(encoding='UTF-8') prog_string=prog_raw.decode(encoding='UTF-8')
prog_list = json.loads(prog_string) prog_list = json.loads(prog_string)
pl=prog_list['programs'][0] pl=prog_list['programs'][0]
return True, pl return True, pl
def read_restdb_old(request_url):
"""
Reads JSON data from W7-X REST API
Returns:
valid: access ok
t: numpy-array of time
signal: numpy-array of requested signals
by H. Thomsen
"""
try:
res = urllib.request.urlopen(request_url)
except urllib.error.URLError as e:
print('read_restdb_old: Error ', e)
return False, 0, -1
else:
signal_raw=res.read()
res.close()
signal_string=signal_raw.decode(encoding='UTF-8')
signal_list = json.loads(signal_string)
signal0=np.array(signal_list['values'])
t=np.array(signal_list['dimensions'])
return True, t, signal0
def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0): def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
""" """
...@@ -236,6 +223,9 @@ def download_NUC_by_times(port,starttime,stoptime,exposure,version=0): ...@@ -236,6 +223,9 @@ def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
except urllib.error.URLError as e: except urllib.error.URLError as e:
print('download_NUC_by_times: Error! ', e) print('download_NUC_by_times: Error! ', e)
goon=False goon=False
except Exception as e:
print('download_NUC_by_times: Error! ', e)
goon=False
if goon: if goon:
n=0 n=0
nuctimes=[0] nuctimes=[0]
...@@ -447,6 +437,9 @@ def download_hot_cold_reference_by_times(port,exposure,starttime=150390720000000 ...@@ -447,6 +437,9 @@ def download_hot_cold_reference_by_times(port,exposure,starttime=150390720000000
except urllib.error.URLError as e: except urllib.error.URLError as e:
print('download_hot_cold_reference_by_times: Error! ',e) print('download_hot_cold_reference_by_times: Error! ',e)
goon=False goon=False
except Exception as e:
print('download_hot_cold_reference_by_times: Error! ',e)
goon=False
if goon: if goon:
COLDtime=0 COLDtime=0
HOTtime=0 HOTtime=0
...@@ -506,6 +499,9 @@ def download_background_by_times(port,starttime,stoptime,exposure,camera_filter= ...@@ -506,6 +499,9 @@ def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=
except urllib.error.URLError as e: except urllib.error.URLError as e:
print('download_background_by_times: Error! ',e) print('download_background_by_times: Error! ',e)
goon=False goon=False
except Exception as e:
print('download_background_by_times: Error! ',e)
goon=False
if goon: if goon:
n=0 n=0
backtimes=[0] backtimes=[0]
...@@ -627,6 +623,8 @@ def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalS ...@@ -627,6 +623,8 @@ def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalS
success=True success=True
except urllib.error.URLError as e: except urllib.error.URLError as e:
print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e)) print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e))
except Exception as e:
print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e))
return success,time,allimages return success,time,allimages
else: else:
try: try:
...@@ -643,6 +641,9 @@ def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalS ...@@ -643,6 +641,9 @@ def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalS
except urllib.error.URLError as e: except urllib.error.URLError as e:
print('download_images_by_times: Error! ',e) print('download_images_by_times: Error! ',e)
return False, 0,-1 return False, 0,-1
except Exception as e:
print('download_images_by_times: Error! ',e)
return False, 0,-1
if fastDL: if fastDL:
def download_raw_images_by_program_via_png(port,program,time_window=0,version=0,threads=1,verbose=0): def download_raw_images_by_program_via_png(port,program,time_window=0,version=0,threads=1,verbose=0):
...@@ -1110,6 +1111,9 @@ def download_last_raw_image_by_time(larchivepath,starttime,stoptime,version=0): ...@@ -1110,6 +1111,9 @@ def download_last_raw_image_by_time(larchivepath,starttime,stoptime,version=0):
except urllib.error.URLError as e: except urllib.error.URLError as e:
print('download_last_raw_image_by_time: Error querrying {0}\n{1}'.format(url,e)) print('download_last_raw_image_by_time: Error querrying {0}\n{1}'.format(url,e))
return False, [-1] return False, [-1]
except Exception as e:
print('download_last_raw_image_by_time: Error querrying {0}\n{1}'.format(url,e))
return False, [-1]
def download_raw_parlog_by_program(port,program,version=0): def download_raw_parlog_by_program(port,program,version=0):
""" """
...@@ -1141,6 +1145,9 @@ def download_raw_parlog_by_times(port,starttime,stoptime,version=0): ...@@ -1141,6 +1145,9 @@ def download_raw_parlog_by_times(port,starttime,stoptime,version=0):
except urllib.error.URLError as e: except urllib.error.URLError as e:
print('download_raw_parlog_by_times: Error! ',e) print('download_raw_parlog_by_times: Error! ',e)
goon=False goon=False
except Exception as E:
print('download_raw_parlog_by_times: Error! ',E)
goon=False
if goon: if goon:
return True, signal_list['dimensions'],signal_list['values'][0]['meta-data'] return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']
...@@ -1175,6 +1182,9 @@ def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0): ...@@ -1175,6 +1182,9 @@ def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0):
except urllib.error.URLError as e: except urllib.error.URLError as e:
print('get_INFRATEC_filter_by_times: Error! ',e) print('get_INFRATEC_filter_by_times: Error! ',e)
goon=False goon=False
except Exception as E:
print('get_INFRATEC_filter_by_times: Error! ',E)
goon=False
if goon: if goon:
return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']['filter'] return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']['filter']
else: else:
...@@ -1787,6 +1797,8 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version= ...@@ -1787,6 +1797,8 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=
for i in range(len(frames)): for i in range(len(frames)):
frames[i]=(frames[i]*FOV).astype(np.float32) frames[i]=(frames[i]*FOV).astype(np.float32)
# if portnr!=50:
if verbose>0: if verbose>0:
print(datetime.datetime.now(),"get_temp_from_raw_by_program: correcting bad pixels") print(datetime.datetime.now(),"get_temp_from_raw_by_program: correcting bad pixels")
frames=IR_tools.correct_images(frames,badpixels) frames=IR_tools.correct_images(frames,badpixels)
...@@ -2098,7 +2110,7 @@ def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_erro ...@@ -2098,7 +2110,7 @@ def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_erro
# if fullbackground: # if fullbackground:
#sbackground=np.zeros(np.shape(images[0]),dtype=np.float32)+background #sbackground=np.zeros(np.shape(images[0]),dtype=np.float32)+background
for i in range(len(images)): for i in range(len(images)):
images[i]=np.clip((images[i]-background),a_min=0,a_max=None).astype(np.uint16) images[i]=np.clip((images[i].astype(np.int32)-background),a_min=0,a_max=None).astype(np.uint16)
# else: # else:
# for i in range(len(images)): # for i in range(len(images)):
# images[i]=images[i]-background # images[i]=images[i]-background
...@@ -2628,7 +2640,7 @@ def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode ...@@ -2628,7 +2640,7 @@ def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode
if verbose>0: if verbose>0:
print(now,"download_heatflux_by_times: heat flux data is not available, creating request") print(now,"download_heatflux_by_times: heat flux data is not available, creating request")
if request: if request:
f=open(heatflux_requestlist_path+str(now.year)+str(now.month)+"_q_requests.txt",'a') f=open(heatflux_requestlist_path+str(now.year)+str(now.month-1)+"_q_requests.txt",'a')
try: try:
programid=AKF_2.get_program_id(tstart) programid=AKF_2.get_program_id(tstart)
except Exception as E: except Exception as E:
...@@ -2711,6 +2723,9 @@ def download_heatflux_scene_model_reference(port,timepoint=None,program=None,ver ...@@ -2711,6 +2723,9 @@ def download_heatflux_scene_model_reference(port,timepoint=None,program=None,ver
except urllib.error.URLError as e: except urllib.error.URLError as e:
print('download_heatflux_scene_model_reference: Error! ',e,larchivepath_par+"/_signal.json?from="+str(timepoint)+"&upto="+str(timepoint)) print('download_heatflux_scene_model_reference: Error! ',e,larchivepath_par+"/_signal.json?from="+str(timepoint)+"&upto="+str(timepoint))
goon=False goon=False
except Exception as E:
print('download_heatflux_scene_model_reference: Error! ',E)
goon=False
if goon: if goon:
timepoint=signal_list['values'][0] timepoint=signal_list['values'][0]
exist,time,frames=download_images_by_times(larchivepath,starttime=timepoint-10,stoptime=int(timepoint+1e8),version=version,verbose=verbose-1) exist,time,frames=download_images_by_times(larchivepath,starttime=timepoint-10,stoptime=int(timepoint+1e8),version=version,verbose=verbose-1)
...@@ -2878,7 +2893,7 @@ def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstar ...@@ -2878,7 +2893,7 @@ def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstar
return False,0,0,0 return False,0,0,0
def extract_heatflux_profile_from_DL(time,images,profile,finger=None,time_window=1, def extract_heatflux_profile_from_DL(time,images,profile,finger=None,time_window=1,
inputchecked=False,verbose=0): inputchecked=False,verbose=0,testmode=False):
""" """
return exist,time,s in m, heat flux in W/m2 return exist,time,s in m, heat flux in W/m2
time_point default (None) returns entire shot. time_point default (None) returns entire shot.
...@@ -2922,7 +2937,7 @@ def extract_heatflux_profile_from_DL(time,images,profile,finger=None,time_window ...@@ -2922,7 +2937,7 @@ def extract_heatflux_profile_from_DL(time,images,profile,finger=None,time_window
else: else:
fingerID=finger*100+profile fingerID=finger*100+profile
### okay finderID is fine, data is given, lets get the mapping ### okay finderID is fine, data is given, lets get the mapping
exist,mapping=download_heatflux_mapping_reference() exist,mapping=download_heatflux_mapping_reference(testmode=testmode)
if exist: if exist:
Fingermap=np.asarray(mapping['Finger_ID'][0],dtype=np.int) Fingermap=np.asarray(mapping['Finger_ID'][0],dtype=np.int)
locs=np.where(Fingermap==fingerID) locs=np.where(Fingermap==fingerID)
...@@ -3177,10 +3192,36 @@ def download_scene_model(port,program=None,timepoint=None,version=0,verbose=0): ...@@ -3177,10 +3192,36 @@ def download_scene_model(port,program=None,timepoint=None,version=0,verbose=0):
#%% general download functions #%% general download functions
""" """
the following functions are copied from W7xrest.read_restdb to remove this dependency the following functions are copied from W7xrest.read_restdb to remove this dependency and are edit to fit into the other functions
@author: thomsen @author: thomsen
""" """
def read_restdb_old(request_url):
"""
Reads JSON data from W7-X REST API
Returns:
valid: access ok
t: numpy-array of time
signal: numpy-array of requested signals
by H. Thomsen
"""
try:
res = urllib.request.urlopen(request_url)
except urllib.error.URLError as e:
print('read_restdb_old: Error ', e)
return False, 0, -1
except Exception as e:
print('read_restdb_old: Error ', e)
return False, 0, -1
else:
signal_raw=res.read()
res.close()
signal_string=signal_raw.decode(encoding='UTF-8')
signal_list = json.loads(signal_string)
signal0=np.array(signal_list['values'])
t=np.array(signal_list['dimensions'])
return True, t, signal0
def read_restdb(request_url): def read_restdb(request_url):
""" """
Reads JSON data from W7-X REST API Reads JSON data from W7-X REST API
...@@ -3204,6 +3245,9 @@ def read_restdb(request_url): ...@@ -3204,6 +3245,9 @@ def read_restdb(request_url):
#print('The server couldn\'t fulfill the request.') #print('The server couldn\'t fulfill the request.')
#print('Error code: ', e.code) #print('Error code: ', e.code)
return False, [0], [-1] return False, [0], [-1]
except Exception as E:
print('read_restdb: Error! ',E)
return False, [0], [-1]
else: else:
# res = urllib.request.urlopen(request_url) # res = urllib.request.urlopen(request_url)
# signal_raw=res.read() # signal_raw=res.read()
...@@ -3262,7 +3306,10 @@ def get_program_from_PID(instring): ...@@ -3262,7 +3306,10 @@ def get_program_from_PID(instring):
prog_list = json.loads(res.read().decode('utf-8')) prog_list = json.loads(res.read().decode('utf-8'))
res.close() res.close()
except urllib.error.URLError as e: except urllib.error.URLError as e:
print('get_program_from_PID: Error opening URL') print('get_program_from_PID: Error opening URL',e)
return False,0
except Exception as E:
print('get_program_from_PID: ERROR appeared',E)
return False,0 return False,0
else: else:
pl=prog_list['programs'] pl=prog_list['programs']
...@@ -3288,24 +3335,30 @@ if __name__=='__main__': ...@@ -3288,24 +3335,30 @@ if __name__=='__main__':
print("local function calling") print("local function calling")
#%% temperature download and plotting example #%% temperature download and plotting example
# port=50#"AEF50"#"AEF51" port=50#"AEF50"#"AEF51"
# prog="20180920.042"#"20171109.021"#"20181010.036" prog='20170927.020'#"20181011.036"#"20181016.037"#"20180920.042"#"20171109.021"#"20181010.036"
# status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_window=[0,2],emi=0.82,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=50,framerate=1) status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_window=[0,4],emi=0.80,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
# status2,time2,images2=download_raw_images_by_program(port,prog,time_window=0.02,verbose=5)
# bla=get_calib_data(50,program=prog,verbose=5)
# success,t,s,profile=extract_temperature_profile_from_DL(port,np.asarray(time-time[0])/1e9,images,profile="TM3h_5_5",verbose=10,reference_time=time[0]) # success,t,s,profile=extract_temperature_profile_from_DL(port,np.asarray(time-time[0])/1e9,images,profile="TM3h_5_5",verbose=10,reference_time=time[0])
# if status: if status:
# plt.figure() plt.figure()
# plt.imshow(images[0],vmin=330,vmax=1000,cmap=exJet) plt.imshow(images[0],vmin=330,vmax=1000,cmap=exJet)
# cb=plt.colorbar() cb=plt.colorbar()
# cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20) cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20)
# cb.ax.tick_params(labelsize=20) cb.ax.tick_params(labelsize=20)
#%% heatflux test #%% heatflux test
# port=10 # port=10
# prog="20171114.039" # prog="20180904.015"
# status,time,images=download_heatflux_by_program(port,prog,timewindow=0.5,threads=1,version=1,verbose=3,givealpha=False,request=True) #
# if status: # for prog in ["20180823.016","20180918.045","20181016.037","20181009.024","20181016.016"]:#"20171207.024","20180814.024","20181016.016","20181010.036"]:
# print(prog,port,"OK") # for port in [10,11,20,21,30,31,40,41,50,51]:
# else: # status,time,images=download_heatflux_by_program(port,prog,time_window=4,threads=4,version=2,verbose=3,givealpha=False,request=True)
# print(prog,port,"missing") ## status,time,load,error=download_divertor_load(port,targetmodule=['all','tm1h'],program=prog,verbose=5,version=3)
# if status:
# print(prog,port,"OK")
# else:
# print(prog,port,"missing")
# plt.figure(),plt.imshow(images[0],vmin=0) # plt.figure(),plt.imshow(images[0],vmin=0)
# plt.figure(),plt.imshow(images[50],vmin=0) # plt.figure(),plt.imshow(images[50],vmin=0)
# status,mapping=download_heatflux_mapping_reference(verbose=4) # status,mapping=download_heatflux_mapping_reference(verbose=4)
......
...@@ -15,7 +15,7 @@ if __name__=='__main__': ...@@ -15,7 +15,7 @@ if __name__=='__main__':
#%% loading data #%% loading data
print(datetime.datetime.now(),"start") print(datetime.datetime.now(),"start")
status,time,images,valid=downIR.get_temp_from_raw_by_program(10,"20180801.017",time_s=2.5,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,verbose=5) status,time,images,valid=downIR.get_temp_from_raw_by_program(10,"20180801.017",time_window=2.5,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,verbose=5)
print('fertig') print('fertig')
#%% plotting data #%% plotting data
......
...@@ -5,7 +5,7 @@ V3.2.0 ...@@ -5,7 +5,7 @@ V3.2.0
@author: holn @author: holn
""" """
import numpy as np import numpy as np
import IR_data_access.downloadversionIRdata as IR import downloadversionIRdata as IR
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import datetime import datetime
...@@ -16,12 +16,12 @@ if __name__=='__main__': ...@@ -16,12 +16,12 @@ if __name__=='__main__':
#%% loading data #%% loading data
print(datetime.datetime.now(),"start") print(datetime.datetime.now(),"start")
status,times,images=IR.download_heatflux_by_program(port,program,time_window=0.9,version=1,threads=1,verbose=5) status,times,images=IR.download_heatflux_by_program(port,program,time_window=6.5,version=2,threads=1,verbose=5,testmode=False)
print('done') print('done')
#%% plotting data #%% plotting data
if status: if status:
status2,time,s,q=IR.extract_heatflux_profile_from_DL((np.asarray(times)-times[0])/1e9,images,profile="TM3h_5_5",time_window=0.5,verbose=5) status2,time,s,q=IR.extract_heatflux_profile_from_DL((np.asarray(times)-times[0])/1e9,images,profile="TM3h_5_5",time_window=None,verbose=5,testmode=False)
plt.figure() plt.figure()
plt.imshow(images[-1]/1e6) plt.imshow(images[-1]/1e6)
plt.clim([0,4]) plt.clim([0,4])
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment