Skip to content
GitLab
Commit 4ee9fab5 authored by Holger Niemann's avatar Holger Niemann
Browse files

Merge branch 'master' into 'peter' 

# Conflicts:
#   downloadversionIRdata.py
parents 9467269b 074ce6e5
Hide whitespace changes
Inline Side-by-side
downloadversionIRdata.py
View file @ 4ee9fab5
......@@ -4,7 +4,7 @@ Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018
last update on Fr Nov 23 15:37:00 2018
Version: 3.2.2
Version: 3.2.4
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
@author: holn
"""
......@@ -149,7 +149,7 @@ def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, ve
print("download_LUT: Error! Camera unknown, stopping here.")
raise Exception
if version==0:
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM")
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM",t_from=time)
if verbose>0:
print("download_LUT: LUT V"+str(version)+" is used")
#time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
......@@ -214,7 +214,7 @@ def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
# NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if version==0:
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM")
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM",t_from=starttime)
try:
res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
signal_list = json.loads(res.read().decode('utf-8'))
......@@ -491,7 +491,7 @@ def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=
stream=portpathdict[OP]["AEF"+str(port)]+"background_"
larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
try:
res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
signal_list = json.loads(res.read().decode('utf-8'))
......@@ -1230,7 +1230,7 @@ def download_raw_parlog_by_times(port,starttime,stoptime,version=0):
"""
OP=IR_tools.get_OP_by_time(time_ns=stoptime)
if version==0:
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG")
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG",t_from=starttime)
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version)
try:
res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
......@@ -1267,7 +1267,7 @@ def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0):
"""
OP=IR_tools.get_OP_by_time(time_ns=starttime)
if version==0:
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG")
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG",t_from=starttime)
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version)
try:
res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
......@@ -1308,7 +1308,7 @@ def get_exposure_by_times(port,starttime,stoptime,version=0):
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/3/exposuretime"+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
def get_camera_temp_by_program(port,program,version=0):
......@@ -1333,7 +1333,7 @@ def get_camera_temp_by_times(port,starttime,stoptime,version=0):
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
dummy=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if dummy[0]:
return dummy
......@@ -1362,7 +1362,7 @@ def get_sensor_temp_by_times(port,starttime,stoptime,version=0):
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
def get_camera_framerate_by_program(port,program,version=0):
......@@ -1388,7 +1388,7 @@ def get_camera_framerate_by_times(port,starttime,stoptime,version=0):
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
......@@ -1414,7 +1414,7 @@ def get_frametype_by_times(port,starttime,stoptime,version=0):
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/5/frametype/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
def get_framenumber_by_program(port,program,version=0):
......@@ -1439,7 +1439,7 @@ def get_framenumber_by_times(port,starttime,stoptime,version=0):
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/4/framenumber/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
def get_average_divertor_TC_for_camtemp(port,starttime,endtime,part="all",TC=0):
......@@ -1962,7 +1962,7 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.
success=True
OP=IR_tools.get_OP_by_time(time_ns=starttime)
if version==0:
version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM")
version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM",program=program)
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
if type(time_window)==list:
tstart=time_window[0]
......@@ -2106,7 +2106,7 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8,
if verbose>0:
print('get_nuced_raw_by_program_fullthreads: prepare loading images by threads')
if version==0:
version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM")
version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM",t_from=starttime)
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
if type(time_window)==list:
tstart=time_window[0]
......@@ -2635,7 +2635,7 @@ def estimate_offset(port,program,plot_it=False,verbose=0):
return -1
def download_heatflux_by_program(port,program,time_window=0,threads=1,testmode=False,
version=0,verbose=0,givealpha=False,request=True):
version=0,verbose=0,givealpha=False,request=True,moffset=0):
"""
port: string or integer
program: string
......@@ -2669,13 +2669,13 @@ def download_heatflux_by_program(port,program,time_window=0,threads=1,testmode=F
tstart=tstart-1.1e9
return download_heatflux_by_times(port=port,tstart=tstart,tend=tend,time_window=0,
testmode=testmode,version=version,verbose=verbose,
request=request)
request=request,moffset=moffset)
else:
print("download_heatflux_by_program: Error! program not found")
return False,0,-1
def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode=False,
version=0,verbose=0,request=True):
version=0,verbose=0,request=True,moffset=0):
"""
port: string or integer
tstart: int64 timestamp in ns from which point the download should start
......@@ -2735,7 +2735,7 @@ def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode
if verbose>0:
print(now,"download_heatflux_by_times: heat flux data is not available")
if request:
f=open(heatflux_requestlist_path+str(now.year)+str(now.month)+"_q_requests.txt",'a')
f=open(heatflux_requestlist_path+str(now.year)+str(now.month+moffset)+"_q_requests.txt",'a')
try:
programid=AKF_2.get_program_id(tstart)
except Exception as E:
......@@ -2761,7 +2761,7 @@ def download_heatflux_mapping_reference(timepoint=None,version=0,testmode=False,
base=archivepath
# OP=IR_tools.get_OP_by_time(time_ns=timepoint)
if version==0:
version=get_latest_version("QRT_IRCAM/Mapping_reference_DATASTREAM",project=project_ana,Test=testmode)
version=get_latest_version("QRT_IRCAM/Mapping_reference_DATASTREAM",project=project_ana,Test=testmode,t_from=timepoint)
larchivepath=base+project_ana+"/"+"QRT_IRCAM/Mapping_reference_DATASTREAM/V"+str(version)+"/0/reference"
### test for the testsample###
# if version==0:
......@@ -3083,9 +3083,8 @@ def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstar
else:
### if the program goes up to here, time okay, finger okay,port okay, lets download it
exist,timo,frames=download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode=False,version=version,verbose=verbose-1)
if exist:
result = extract_heatflux_profile_from_DL(time=(np.asarray(timo)-t1)/1e9,images=frames,profile=int(fingerID%100),finger=int(fingerID/100),time_window=None,inputchecked=inputcheck,verbose=verbose-1)
return result
if exist:
return extract_heatflux_profile_from_DL(time=(np.asarray(timo)-t1)/1e9,images=frames,profile=int(fingerID%100),finger=int(fingerID/100),time_window=timepoint,inputchecked=inputcheck,verbose=verbose-1)
else:
if verbose>0:
print("get_heatflux_profile: heatflux data could not be downloaded")
......@@ -3536,25 +3535,26 @@ if __name__=='__main__':
print("local function calling")
#%% temperature download and plotting example
# port=50#"AEF50"#"AEF51"
# prog='20170927.020'#"20181011.036"#"20181016.037"#"20180920.042"#"20171109.021"#"20181010.036"
# status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_window=[0,4],emi=0.80,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
## status2,time2,images2=download_raw_images_by_program(port,prog,time_window=0.02,verbose=5)
## bla=get_calib_data(50,program=prog,verbose=5)
## success,t,s,profile=extract_temperature_profile_from_DL(port,np.asarray(time-time[0])/1e9,images,profile="TM3h_5_5",verbose=10,reference_time=time[0])
# if status:
# plt.figure()
# plt.imshow(images[0],vmin=330,vmax=1000,cmap=exJet)
# cb=plt.colorbar()
# cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20)
# cb.ax.tick_params(labelsize=20)
port=31#"AEF50"#"AEF51"'20171114.053'#
prog="20180911.008"#"20181011.033"#'20171122.035'#'20181011.010'#'20180823.037'#'20170927.020'#"20181011.036"#"20181016.037"#"20180920.042"#"20171109.021"#"20181010.036"
status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_window=[5,5.1],emi=0.80,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
# status2,time2,images2=download_raw_images_by_program(port,prog,time_window=0.02,verbose=5)
# bla=get_calib_data(50,program=prog,verbose=5)
# success,t,s,profile=extract_temperature_profile_from_DL(port,np.asarray(time-time[0])/1e9,images,profile="TM3h_5_5",verbose=10,reference_time=time[0])
if status:
plt.figure()
plt.imshow(images[-1],vmin=330,vmax=1000,cmap=exJet)
cb=plt.colorbar()
cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20)
cb.ax.tick_params(labelsize=20)
plt.title("AEF{0}".format(port))
#%% heatflux test
port=10
prog="20180904.015"
# for prog in ["20180823.016","20180918.045","20181016.037","20181009.024","20181016.016"]:#"20171207.024","20180814.024","20181016.016","20181010.036"]:
# port=10
# prog="20180904.015"
#
# for prog in ["20171207.016"]:#"20180821.012"]:#"20180823.016","20180918.045","20181016.037","20181009.024","20181016.016"]:#"20171207.024","20180814.024","20181016.016","20181010.036"]:
# for port in [10,11,20,21,30,31,40,41,50,51]:
# status,time,images=download_heatflux_by_program(port,prog,time_window=4,threads=4,version=2,verbose=3,givealpha=False,request=True)
# status,time,images=download_heatflux_by_program(port,prog,time_window=0.9,threads=4,version=2,verbose=3,givealpha=False,request=True)
## status,time,load,error=download_divertor_load(port,targetmodule=['all','tm1h'],program=prog,verbose=5,version=3)
# if status:
# print(prog,port,"OK")
......@@ -3566,9 +3566,10 @@ if __name__=='__main__':
# test=get_heatflux_profile(20,1605,timepoint=1,program="20171109.008",verbose=4)
#%% loads test
# port=21
# prog="20171114.052"
# status,time,load,error=download_divertor_load(port,targetmodule=['all','tm1h'],program=prog,verbose=5,debug=True)
# port=10
# prog="20171207.016"#"20181011.015"#"20171114.052"
# for port in [10,11,20,21,30,31,40,41,50,51]:
# status,time,load,error=download_divertor_load(port,targetmodule=['all','tm1h'],program=prog,verbose=5)
#%% scene model test
# port=10
# exist,model=download_scene_model(port)
......
plot_heatflux_example.py
View file @ 4ee9fab5
......@@ -9,8 +9,8 @@ import downloadversionIRdata as IR
import matplotlib.pyplot as plt
import datetime
port=10 # 20
program="20171207.022" # "20181004.012"
port=51 # 20
program="20180927.017"#"20180904.007"#"20171207.022" # "20181004.012"
if __name__=='__main__':
......
setup.py 0 → 100644
View file @ 4ee9fab5
from setuptools import setup, find_packages
setup(
name = 'ir-data-access',
version = '3.2.4',
author = 'Holger Niemann, Peter Drewelow',
author_email = 'holger.niemann@ipp.mpg.de',
description = 'Access Frontend for IR camera data',
py_modules = [
'downloadversionIRdata',
'IR_image_tools',
'IR_config_constants',
'IR_image_tools',
'plot_IR_data' ,
'Create_HDF5_from_archive_data',
'plot_heatflux_example'
],
data_files=[
('',['upload_config','CHANGELOG','ToDo.txt']),
('data',['data/AEF10_coldframes_background_fails_real.txt',
'data/AEF11_coldframes_background_fails_real.txt',
'data/AEF20_coldframes_background_fails_real.txt',
'data/AEF21_coldframes_background_fails_real.txt',
'data/AEF30_coldframes_background_fails_real.txt',
'data/AEF31_coldframes_background_fails_real.txt',
'data/AEF40_coldframes_background_fails_real.txt',
'data/AEF41_coldframes_background_fails_real.txt',
'data/AEF51_coldframes_background_fails_real.txt'])
],
install_requires = [
'archivedb>=0.2.0'
]
);
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment