diff --git a/downloadversionIRdata.py b/downloadversionIRdata.py index e95ee4f3a9e8c00534a5a7fb535780b5b9cfa49f..16632b1d5101e1de79720881eecf1534e59c58fc 100644 --- a/downloadversionIRdata.py +++ b/downloadversionIRdata.py @@ -3352,7 +3352,7 @@ def get_temp_from_raw_by_program(portnr, program, time_window=0, emi=0.8, T_vers INPUT ------ - portnr: integer + portnr: integer or string port number of the AEF port of the camera program: string program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' @@ -3381,7 +3381,10 @@ def get_temp_from_raw_by_program(portnr, program, time_window=0, emi=0.8, T_vers if True, the data will be loaded from the test archive framerate: string or integer, optional, default 'max' sets the framerate to downsample the data. normal rate is 100. \n - works only if w7xarchive is available! + works only if w7xarchive is available! + caching: boolean, default False + activates the caching option, which will save the downloaded data locally for later use to save the time for download\n + WARNING: this can easily fill up the harddisk, since the temperature and heatflux data is up to 2GB per camera and program RESULT ------ exist: boolean @@ -3427,6 +3430,8 @@ def get_temp_from_raw_by_program(portnr, program, time_window=0, emi=0.8, T_vers # raise Exception("get_temp_from_raw_by_program: the given port is neither a number or a valid String!") logging.warning("get_temp_from_raw_by_program: the given port is neither a number or a valid String!") return False, [0], [0] + if caching: + return load_cache_file(typ="temperature",port=portnr,program=program,time_window=time_window,framerate=framerate,verbose=verbose) if FLIR: # --- Glens FLIR camera --- # @@ -4729,6 +4734,12 @@ def download_heatflux_by_program(port, program, time_window=0, testmode=False, it sets the timewindow start automatically to t1 if active request: boolean, default True switch to turn on or off the request of data if the data is not available in the database + framerate: string or integer, optional, default 'max', not functional for this function up to now + sets the framerate to downsample the data. normal rate is 100. \n + works only if w7xarchive is available! + caching: boolean, default False + activates the caching option, which will save the downloaded data locally for later use to save the time for download\n + WARNING: this can easily fill up the harddisk, since the temperature and heatflux data is up to 2GB per camera and program RESULT ------ (returns downlad_heatflux_by_times) @@ -4808,6 +4819,12 @@ def download_heatflux_by_times(port, tstart, tend, time_window=0, testmode=False switch to turn on or off the request of data if the data is not available in the database request_ALL: boolean, optional, default False switch to turn on the request of data for all AEF ports + framerate: string or integer, optional, default 'max', not functional for this function up to now + sets the framerate to downsample the data. normal rate is 100. \n + works only if w7xarchive is available! + caching: boolean, default False + activates the caching option, which will save the downloaded data locally for later use to save the time for download\n + WARNING: this can easily fill up the harddisk, since the temperature and heatflux data is up to 2GB per camera and program RESULT ------ exist: boolean @@ -4846,6 +4863,9 @@ def download_heatflux_by_times(port, tstart, tend, time_window=0, testmode=False if not goon: raise Exception("download_heatflux_by_times: the given port is neither a number or a valid String!") else: + if caching: + program = AKF_2.get_program_id(t0=tstart,dt=1) + return load_cache_file(typ="heatflux",port=port,program=program,time_window=time_window,framerate=framerate,verbose=verbose) if FLIR: if verbose > 0: print("download_heatflux_by_times: FLIR heatflux is at the moment unsupported!") @@ -6216,8 +6236,7 @@ def download_scene_model(port, program=None, timepoint=None, version=0, testmode INPUT ----- port: integer, float, string or list of int, float or strings - array of heat fluxes from THEODOR on the profiles defined - in the IR mapping; can be 2D (one divertor), or 3D (multiple divertor modules) + program : string, optional Program ID of an W7-X Program in form of "20160224.025" to indicate the validation window timepoint: int64, optional @@ -6395,18 +6414,39 @@ def check_path_and_make_folders(activate, path, sep='', folder=''): #============================================================================== import pickle #here the new implementation -def load_cache_file(typ,port,program,time_window,framerate,caching=False,verbose=0): +def load_cache_file(typ,port,program,time_window,framerate='max',verbose=0): """ - supports temperature and heatflux + supports temperature and heatflux\n + will try to load a cache file, it not succesfull it will load the data and create a cache file.\n - Note + INPUT + ----- + typ: string + kind of data, for temperature data "temperature", for heat flux data "heatflux" + port: integer, float, string + port describtion of the camera + program : string, optional + Program ID of an W7-X Program in form of "20160224.025" to indicate the validation window + time_window: integer, float or list/numpy array of interger/float, + time window for the heat flux, single value: t1 until so many seconds, two values, t1+ first value until t1+ second value + framerate: string or integer, optional, default 'max', not functional for this function up to now + sets the framerate to downsample the data. normal rate is 100. \n + works only if w7xarchive is available! + verbose: integer, optional + feedback level (details of print messages) + RESULT + ------ + dat: function return + returns the result from either get_temp_from_raw_by_program or downoad_heatflux_by_program - the maximum framerate definition must be adjusted if the IR cameras get faster than 100 Hz. + NOTES ----- - case of several smaller intervalls are not covered, e.g. 1 to 2 seconds is in cache and 4 to 5 seconds is requested - at the moment only one file per program and camera is supported + the maximum framerate definition must be adjusted if the IR cameras get faster than 100 Hz.\n + + case of several smaller intervalls are not covered, e.g. 1 to 2 seconds is in cache and 4 to 5 seconds is requested + at the moment only one file per program and camera is supported """ framerate = str(framerate) if type(port) == str: @@ -6445,9 +6485,7 @@ def load_cache_file(typ,port,program,time_window,framerate,caching=False,verbose fcase = 0 else:# it does not fit, requested framerate is faster than available one, file has to be replaced FR = False - fcase = 0 - - + fcase = 0 ### check time window. is the requested time window in the cache or partly in the cache? ##cases: 1 it fits perfect, 2 the requested is in the available, 3 the requested is partly in the available, 0 no fitting, 5 separated file needed if "[" in t_window: @@ -6501,14 +6539,112 @@ def load_cache_file(typ,port,program,time_window,framerate,caching=False,verbose dat[1]=dummy1 dat[2]=dummy2 return dat - else:#okay available time is larger - print("to be implemented") - dat[0]=False - return dat + else:#okay available time is larger + prog = get_program_from_PID(program) + if prog[0] and dat[0]: + t_start = prog[1]['trigger']['1'][0] + t_vec_s = (np.asarray(dat[1])-t_start)/1e6 + TID1 = IR_tools.find_nearest(t_vec_s, time_window2[0]) + TID2 = IR_tools.find_nearest(t_vec_s, time_window2[1]) + if fcase == 1: + dat[1] = dat[1][TID1:TID2] + dat[2] = dat[2][TID1:TID2] + return dat + else: #the framerate is to fast!, case 2, both should be now numbers + fratio = f_rate / framerate + leni = len(dat[1][TID1:TID2]) + dummy1 = []#dat[1] + dummy2 = []#dat[2] + step = int(leni/fratio) + for i in range(step): + dummy1.append(dat[1][TID1:TID2][int(i*fratio)]) + dummy2.append(dat[2][TID1:TID2][int(i*fratio)]) + dat[1]=dummy1 + dat[2]=dummy2 + return dat + else: + if prog[0]: + cachename = f"IR_{typ}_{program}_{portname}_{t_window}_{f_rate}.cache" + logging.warning("load_cache_file: cache file is trash, file: {cachename}") + else: + logging.warning("load_cache_file: cannot find the program") + return dat else:#case 3, partly available data, need to extend the data and resave it - print("to be implemented") - dat[0]=False - return dat + + # two possible cases, requested time window starts bevor the available one or extends longer + # case 1: _______ case 2 ____________ case 3(ignored case) _______________ + # request | | | | | new complete | | + # cache |____|__| |______| download |___| + prog = get_program_from_PID(program) + if prog[0] and dat[0]: + t_start = prog[1]['trigger']['1'][0] + t_vec_s = (np.asarray(dat[1])-t_start)/1e6 + TID1 = IR_tools.find_nearest(t_vec_s, time_window2[0]) + TID2 = IR_tools.find_nearest(t_vec_s, time_window2[1]) + cas1 = False + if t_vec_s[TID1] > time_window2[0]:#case 1 as pictured above + cas1 = True + t_win = [time_window2[0],t_vec_s[TID1]] + t_win2 = [time_window2[0],t_vec_s[TID2]] + cachename = f"IR_{typ}_{program}_{portname}_{t_window}_{f_rate}.cache" + cachename_new = f"IR_{typ}_{program}_{portname}_{t_win2}_{f_rate}.cache" + else:#case 2 + cas1 = False + t_win = [t_vec_s[TID2],time_window2[1]] + t_win2 = [t_vec_s[TID1],time_window2[1]] + cachename = f"IR_{typ}_{program}_{portname}_{t_window}_{f_rate}.cache" + cachename_new = f"IR_{typ}_{program}_{portname}_{t_win2}_{f_rate}.cache" + if typ == "temperature": + threads = os.cpu_count()*2 + dat2 = get_temp_from_raw_by_program(port, program, time_window=t_win, threads=threads, verbose=int(verbose), framerate=framerate, caching=False) + elif typ == "heatflux": + dat2 = download_heatflux_by_program(port, program, time_window=t_win, framerate=framerate, verbose=verbose, caching=False) + else: + raise Exception("load_cache_file: unknown data typ!") + #merge dat and dat 2 + if dat2[0] and dat[0]: + if cas1: + dat[1]=dat2[1]+dat[1] + dat[2]=dat2[2]+dat[2] + else: + dat[1]=dat[1]+dat2[1] + dat[2]=dat[2]+dat2[2] + #resave cache + os.remove(cpath+"\\"+cachename) + with open(cachename_new, "wb") as f: + pickle.dump(dat, f) + if verbose>0: + print(f"{program}: returning newly calculated {typ} data, cached for later use") + #extract framerate if needed + t_vec_s = (np.asarray(dat[1])-t_start)/1e6 + TID1 = IR_tools.find_nearest(t_vec_s, time_window2[0]) + TID2 = IR_tools.find_nearest(t_vec_s, time_window2[1]) + if fcase == 1: + dat[1] = dat[1][TID1:TID2] + dat[2] = dat[2][TID1:TID2] + return dat + else: #the framerate is to fast!, case 2, both should be now numbers + fratio = f_rate / framerate + leni = len(dat[1][TID1:TID2]) + dummy1 = []#dat[1] + dummy2 = []#dat[2] + step = int(leni/fratio) + for i in range(step): + dummy1.append(dat[1][TID1:TID2][int(i*fratio)]) + dummy2.append(dat[2][TID1:TID2][int(i*fratio)]) + dat[1]=dummy1 + dat[2]=dummy2 + return dat + else: + logging.warning("load_cache_file: cannot merge old and new cache file because one of them is trash") + + else: + if prog[0]: + cachename = f"IR_{typ}_{program}_{portname}_{t_window}_{f_rate}.cache" + logging.warning("load_cache_file: cache file is trash, file: {cachename}") + else: + logging.warning("load_cache_file: cannot find the program") + return dat else:#delete old file and replace it with the new one if verbose>0: print("load_cache_file: file found but it does not match the requirements, overwriting the old file now")