Skip to content
GitLab
Commit 02ec8432 authored by Holger Niemann's avatar Holger Niemann
Browse files

UPDATE V3.6.0: missing cases for caching added and function integrated into... 

UPDATE V3.6.0: missing cases for caching added and function integrated into get_temp_from_raw_by_program and download_heatflux_by_program (or times), need to be tested
parent 3e2e04e6
Hide whitespace changes
Inline Side-by-side
downloadversionIRdata.py
View file @ 02ec8432
......@@ -3352,7 +3352,7 @@ def get_temp_from_raw_by_program(portnr, program, time_window=0, emi=0.8, T_vers
INPUT
------
portnr: integer
portnr: integer or string
port number of the AEF port of the camera
program: string
program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016'
......@@ -3381,7 +3381,10 @@ def get_temp_from_raw_by_program(portnr, program, time_window=0, emi=0.8, T_vers
if True, the data will be loaded from the test archive
framerate: string or integer, optional, default 'max'
sets the framerate to downsample the data. normal rate is 100. \n
works only if w7xarchive is available!
works only if w7xarchive is available!
caching: boolean, default False
activates the caching option, which will save the downloaded data locally for later use to save the time for download\n
WARNING: this can easily fill up the harddisk, since the temperature and heatflux data is up to 2GB per camera and program
RESULT
------
exist: boolean
......@@ -3427,6 +3430,8 @@ def get_temp_from_raw_by_program(portnr, program, time_window=0, emi=0.8, T_vers
# raise Exception("get_temp_from_raw_by_program: the given port is neither a number or a valid String!")
logging.warning("get_temp_from_raw_by_program: the given port is neither a number or a valid String!")
return False, [0], [0]
if caching:
return load_cache_file(typ="temperature",port=portnr,program=program,time_window=time_window,framerate=framerate,verbose=verbose)
if FLIR:
# --- Glens FLIR camera ---
#
......@@ -4729,6 +4734,12 @@ def download_heatflux_by_program(port, program, time_window=0, testmode=False,
it sets the timewindow start automatically to t1 if active
request: boolean, default True
switch to turn on or off the request of data if the data is not available in the database
framerate: string or integer, optional, default 'max', not functional for this function up to now
sets the framerate to downsample the data. normal rate is 100. \n
works only if w7xarchive is available!
caching: boolean, default False
activates the caching option, which will save the downloaded data locally for later use to save the time for download\n
WARNING: this can easily fill up the harddisk, since the temperature and heatflux data is up to 2GB per camera and program
RESULT
------
(returns downlad_heatflux_by_times)
......@@ -4808,6 +4819,12 @@ def download_heatflux_by_times(port, tstart, tend, time_window=0, testmode=False
switch to turn on or off the request of data if the data is not available in the database
request_ALL: boolean, optional, default False
switch to turn on the request of data for all AEF ports
framerate: string or integer, optional, default 'max', not functional for this function up to now
sets the framerate to downsample the data. normal rate is 100. \n
works only if w7xarchive is available!
caching: boolean, default False
activates the caching option, which will save the downloaded data locally for later use to save the time for download\n
WARNING: this can easily fill up the harddisk, since the temperature and heatflux data is up to 2GB per camera and program
RESULT
------
exist: boolean
......@@ -4846,6 +4863,9 @@ def download_heatflux_by_times(port, tstart, tend, time_window=0, testmode=False
if not goon:
raise Exception("download_heatflux_by_times: the given port is neither a number or a valid String!")
else:
if caching:
program = AKF_2.get_program_id(t0=tstart,dt=1)
return load_cache_file(typ="heatflux",port=port,program=program,time_window=time_window,framerate=framerate,verbose=verbose)
if FLIR:
if verbose > 0:
print("download_heatflux_by_times: FLIR heatflux is at the moment unsupported!")
......@@ -6216,8 +6236,7 @@ def download_scene_model(port, program=None, timepoint=None, version=0, testmode
INPUT
-----
port: integer, float, string or list of int, float or strings
array of heat fluxes from THEODOR on the profiles defined
in the IR mapping; can be 2D (one divertor), or 3D (multiple divertor modules)
program : string, optional
Program ID of an W7-X Program in form of "20160224.025" to indicate the validation window
timepoint: int64, optional
......@@ -6395,18 +6414,39 @@ def check_path_and_make_folders(activate, path, sep='', folder=''):
#==============================================================================
import pickle
#here the new implementation
def load_cache_file(typ,port,program,time_window,framerate,caching=False,verbose=0):
def load_cache_file(typ,port,program,time_window,framerate='max',verbose=0):
"""
supports temperature and heatflux
supports temperature and heatflux\n
will try to load a cache file, it not succesfull it will load the data and create a cache file.\n
Note
INPUT
-----
typ: string
kind of data, for temperature data "temperature", for heat flux data "heatflux"
port: integer, float, string
port describtion of the camera
program : string, optional
Program ID of an W7-X Program in form of "20160224.025" to indicate the validation window
time_window: integer, float or list/numpy array of interger/float,
time window for the heat flux, single value: t1 until so many seconds, two values, t1+ first value until t1+ second value
framerate: string or integer, optional, default 'max', not functional for this function up to now
sets the framerate to downsample the data. normal rate is 100. \n
works only if w7xarchive is available!
verbose: integer, optional
feedback level (details of print messages)
RESULT
------
dat: function return
returns the result from either get_temp_from_raw_by_program or downoad_heatflux_by_program
the maximum framerate definition must be adjusted if the IR cameras get faster than 100 Hz.
NOTES
-----
case of several smaller intervalls are not covered, e.g. 1 to 2 seconds is in cache and 4 to 5 seconds is requested
at the moment only one file per program and camera is supported
the maximum framerate definition must be adjusted if the IR cameras get faster than 100 Hz.\n
case of several smaller intervalls are not covered, e.g. 1 to 2 seconds is in cache and 4 to 5 seconds is requested
at the moment only one file per program and camera is supported
"""
framerate = str(framerate)
if type(port) == str:
......@@ -6445,9 +6485,7 @@ def load_cache_file(typ,port,program,time_window,framerate,caching=False,verbose
fcase = 0
else:# it does not fit, requested framerate is faster than available one, file has to be replaced
FR = False
fcase = 0
fcase = 0
### check time window. is the requested time window in the cache or partly in the cache?
##cases: 1 it fits perfect, 2 the requested is in the available, 3 the requested is partly in the available, 0 no fitting, 5 separated file needed
if "[" in t_window:
......@@ -6501,14 +6539,112 @@ def load_cache_file(typ,port,program,time_window,framerate,caching=False,verbose
dat[1]=dummy1
dat[2]=dummy2
return dat
else:#okay available time is larger
print("to be implemented")
dat[0]=False
return dat
else:#okay available time is larger
prog = get_program_from_PID(program)
if prog[0] and dat[0]:
t_start = prog[1]['trigger']['1'][0]
t_vec_s = (np.asarray(dat[1])-t_start)/1e6
TID1 = IR_tools.find_nearest(t_vec_s, time_window2[0])
TID2 = IR_tools.find_nearest(t_vec_s, time_window2[1])
if fcase == 1:
dat[1] = dat[1][TID1:TID2]
dat[2] = dat[2][TID1:TID2]
return dat
else: #the framerate is to fast!, case 2, both should be now numbers
fratio = f_rate / framerate
leni = len(dat[1][TID1:TID2])
dummy1 = []#dat[1]
dummy2 = []#dat[2]
step = int(leni/fratio)
for i in range(step):
dummy1.append(dat[1][TID1:TID2][int(i*fratio)])
dummy2.append(dat[2][TID1:TID2][int(i*fratio)])
dat[1]=dummy1
dat[2]=dummy2
return dat
else:
if prog[0]:
cachename = f"IR_{typ}_{program}_{portname}_{t_window}_{f_rate}.cache"
logging.warning("load_cache_file: cache file is trash, file: {cachename}")
else:
logging.warning("load_cache_file: cannot find the program")
return dat
else:#case 3, partly available data, need to extend the data and resave it
print("to be implemented")
dat[0]=False
return dat
# two possible cases, requested time window starts bevor the available one or extends longer
# case 1: _______ case 2 ____________ case 3(ignored case) _______________
# request | | | | | new complete | |
# cache |____|__| |______| download |___|
prog = get_program_from_PID(program)
if prog[0] and dat[0]:
t_start = prog[1]['trigger']['1'][0]
t_vec_s = (np.asarray(dat[1])-t_start)/1e6
TID1 = IR_tools.find_nearest(t_vec_s, time_window2[0])
TID2 = IR_tools.find_nearest(t_vec_s, time_window2[1])
cas1 = False
if t_vec_s[TID1] > time_window2[0]:#case 1 as pictured above
cas1 = True
t_win = [time_window2[0],t_vec_s[TID1]]
t_win2 = [time_window2[0],t_vec_s[TID2]]
cachename = f"IR_{typ}_{program}_{portname}_{t_window}_{f_rate}.cache"
cachename_new = f"IR_{typ}_{program}_{portname}_{t_win2}_{f_rate}.cache"
else:#case 2
cas1 = False
t_win = [t_vec_s[TID2],time_window2[1]]
t_win2 = [t_vec_s[TID1],time_window2[1]]
cachename = f"IR_{typ}_{program}_{portname}_{t_window}_{f_rate}.cache"
cachename_new = f"IR_{typ}_{program}_{portname}_{t_win2}_{f_rate}.cache"
if typ == "temperature":
threads = os.cpu_count()*2
dat2 = get_temp_from_raw_by_program(port, program, time_window=t_win, threads=threads, verbose=int(verbose), framerate=framerate, caching=False)
elif typ == "heatflux":
dat2 = download_heatflux_by_program(port, program, time_window=t_win, framerate=framerate, verbose=verbose, caching=False)
else:
raise Exception("load_cache_file: unknown data typ!")
#merge dat and dat 2
if dat2[0] and dat[0]:
if cas1:
dat[1]=dat2[1]+dat[1]
dat[2]=dat2[2]+dat[2]
else:
dat[1]=dat[1]+dat2[1]
dat[2]=dat[2]+dat2[2]
#resave cache
os.remove(cpath+"\\"+cachename)
with open(cachename_new, "wb") as f:
pickle.dump(dat, f)
if verbose>0:
print(f"{program}: returning newly calculated {typ} data, cached for later use")
#extract framerate if needed
t_vec_s = (np.asarray(dat[1])-t_start)/1e6
TID1 = IR_tools.find_nearest(t_vec_s, time_window2[0])
TID2 = IR_tools.find_nearest(t_vec_s, time_window2[1])
if fcase == 1:
dat[1] = dat[1][TID1:TID2]
dat[2] = dat[2][TID1:TID2]
return dat
else: #the framerate is to fast!, case 2, both should be now numbers
fratio = f_rate / framerate
leni = len(dat[1][TID1:TID2])
dummy1 = []#dat[1]
dummy2 = []#dat[2]
step = int(leni/fratio)
for i in range(step):
dummy1.append(dat[1][TID1:TID2][int(i*fratio)])
dummy2.append(dat[2][TID1:TID2][int(i*fratio)])
dat[1]=dummy1
dat[2]=dummy2
return dat
else:
logging.warning("load_cache_file: cannot merge old and new cache file because one of them is trash")
else:
if prog[0]:
cachename = f"IR_{typ}_{program}_{portname}_{t_window}_{f_rate}.cache"
logging.warning("load_cache_file: cache file is trash, file: {cachename}")
else:
logging.warning("load_cache_file: cannot find the program")
return dat
else:#delete old file and replace it with the new one
if verbose>0:
print("load_cache_file: file found but it does not match the requirements, overwriting the old file now")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment