diff --git a/CHANGELOG b/CHANGELOG index f6e346cea7556fe4bc4e5dd4c5ab0932d66b3b8a..99d9c6eb984a6a2e7274f95096db2cb1ca8e5c27 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,19 +1,40 @@ <<<<<<< HEAD +XX.06.2020: Update to V3.5.0 + - add download maximum heat flux and wetted area +16.06.2020: Update to V3.4.1 + - updated and improved documentation + - include chache function and a request for all ports for heat flux and divertor load 08.02.2020: Update to V3.4.0 - Bugfix: download_images_by_times produzierte doppelte Werte im Zeit und Bildvektor - - + - Bugfix: download divertor load gave print back that request was made, although no request was made (fixed) + - Bugfix: correction of version bug from last commit and change of default value for the profile extraction functions to None + - Readme added Update to V3.3.3: - - + - added function for peaking factor and strike line width calculation in tools + - bugfix: bug fix get_trigger_from_PID which caused an error in get_latest_version if it was used for non IR data Update to V3.3.2: - - + - extended feedback for download_heatflux + - changes of wetted area calculation + - changes of wetted area calculation + - bugfix: get_trigger_PID, get_finger and during import (see issues 6,7) Update to V3.3.1: - - + - bugfix: FOV for AEF50 was not working correctly Update to V3.3.0: - - + - IR_config_constants: added path to Test archive and use this variable in downloadversionIRdata when building an archive path + - downloadversionIRdata: consistent use of testmode parameter + - downloadversionIRdata: added testmode to all reading routines allowing reading from Test archive + - IR_image_tools: no longer sets working directory to script file location + - IR_config_constants: sets parameter_file_path as absolute path + - downloadversionIRdata: new function get_trigger_from_PID() wraps get_program_from_PID() and allows getting timestamps for lab data + - downloadversionIRdata: download_images_by_time_via_png() now uses nanosecond timestamps to call AKF_2.get_time_intervals() + - downloadversionIRdata: disable DeprecationWarning in import of archivedb (timezone issue on Windows) + - downloadversionIRdata: get_NUC_by_times() uses the reference cold frame in case no cold frame (NUC frame) was found for this time + - downloadversionIRdata: unified use of stoptime instead of endtime + - downloadversionIRdata: expanded testing sections (profile, coldframe, timestamps and scene model) Update to V3.2.5: - + - remove of old, not needed function and dependencies, update of download_background function Update to V3.2.4: - + - smaller bugfixes and setup.py got added Update to V3.2.3: - bugfix: finding right version for meta informations Update to V3.2.2: @@ -84,7 +105,8 @@ bugfixes in downloadversionIRdata: - fixed: wrong time intervall for TC divertor data Versions: -V3.4.0: chaching functionality included, request for all cameras included + Bugfixes +V3.5.0: download of wetted area, peak loads for different targets and +V3.4.1: chaching functionality included, request for all cameras included + Bugfixes V3.3.0: code-cleaning, unification of variable names, new functions: get_trigger_from_PID, V3.2.0: download of scene models possible, temperature profiles can be extracted, downsampled temperature images available V3.1.0: add of divertor loads in the upload and the download diff --git a/IR_image_tools.py b/IR_image_tools.py index 5a5b6c730cac9c8eec6e205fddbdf9a6007b0700..ce8eefcb760143f010e964aa7f919405a9524f6a 100644 --- a/IR_image_tools.py +++ b/IR_image_tools.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ Created on Wed May 9 14:56:32 2018 -Version: 3.3.3 +Version: 3.4.0 @author: Holger Niemann, Peter Drewelow, Yu Gao mainly to clean up the downloadversionIRdata code @@ -24,14 +24,17 @@ import datetime def get_OP_by_time(time_ns=None, shot_no=None, program_str=None): '''Derives operation phase (OP) of W7-X based on either: a nanosacond time stamp, a MDSplus style shot no. or a program ID. - IN: - time_ns - integer of nanosecond time stamp, + + INPUT + ----- + time_ns: - integer of nanosecond time stamp, e.g. 1511972727249834301 (OPTIONAL) shot_no - integer of MDSplus style shot number, e.g. 171207022 (OPTIONAL) program_str - string of CoDaQ ArchiveDB style prgram number or date, e.g. '20171207.022' or '20171207' (OPTIONAL) - RETURN: + RESULT + ------ conn - MDSplus connection object, to be used in e.g. 1511972727249834301 read_MDSplus_image_simple(), read_MDSplus_metadata() ''' @@ -64,6 +67,16 @@ def get_OP_by_time(time_ns=None, shot_no=None, program_str=None): return OP def bestimmtheitsmass_general(data, fit): + """ + INPUT + ------ + + RESULT + ------ + + NOTE + ------ + """ R = 0 if len(fit) == len(data): mittel = np.sum(data)/len(data) @@ -74,6 +87,16 @@ def bestimmtheitsmass_general(data, fit): return R def bestimmheitsmass_linear(data, fit, debugmode=False): + """ + INPUT + ------ + + RESULT + ------ + + NOTE + ------ + """ R2 = 0 if len(fit) == len(data): mittel_D = np.mean(data)#np.sum(data)/len(data) @@ -87,12 +110,32 @@ def bestimmheitsmass_linear(data, fit, debugmode=False): return R2 def quad_abweich_mittel(data, mittel): + """ + INPUT + ------ + + RESULT + ------ + + NOTE + ------ + """ R = 0 for i in data: R = R+(i-mittel)**2 return R def quad_abweich(data, fit): + """ + INPUT + ------ + + RESULT + ------ + + NOTE + ------ + """ R = 0 if len(fit) == len(data): for i in range(len(data)): @@ -102,6 +145,16 @@ def quad_abweich(data, fit): return R def find_nearest(array, value): + """ + INPUT + ------ + + RESULT + ------ + + NOTE + ------ + """ #a=array a = [x - value for x in array] mini = np.min(np.abs(a)) @@ -112,6 +165,15 @@ def find_nearest(array, value): def check_coldframe(coldframe, references=None, threshold=0.5, plot_it=False): ''' return true/false and the quality factor + + INPUT + ------ + + RESULT + ------ + + NOTE + ------ ''' shapi = np.shape(coldframe) ##function (np.arange(0,768)-384)**(2)/900-50 @@ -147,6 +209,14 @@ def check_coldframe(coldframe, references=None, threshold=0.5, plot_it=False): def check_coldframe_by_refframe(coldframe, reference_frame, threshold=0.8, plot_it=False): ''' + INPUT + ------ + + RESULT + ------ + + NOTE + ------ ''' references = [] shapi = np.shape(reference_frame) @@ -158,6 +228,15 @@ def check_coldframe_by_refframe(coldframe, reference_frame, threshold=0.8, plot_ def check_backgroundframe(backgroundframe, threshold=50): ''' return true or false + + INPUT + ------ + + RESULT + ------ + + NOTE + ------ ''' shapi = np.shape(backgroundframe) valid = True @@ -181,6 +260,15 @@ def read_bad_pixels_from_file(port, shot_no=None, program=None, time_ns=None): OUT bad_pixle_list - list of tuples (row,column) of pixel coordinates as integer + + INPUT + ------ + + RESULT + ------ + + NOTE + ------ ''' if shot_no is not None: OP = get_OP_by_time(shot_no=shot_no) @@ -204,6 +292,15 @@ def find_outlier_pixels(frame, tolerance=3, plot_it=False):#worry_about_edges=Tr ''' This function finds the bad pixels in a 2D dataset. Tolerance is the number of standard deviations used for cutoff. + + INPUT + ------ + + RESULT + ------ + + NOTE + ------ ''' frame = np.array(frame)#, dtype=int) from scipy.ndimage import median_filter @@ -242,6 +339,14 @@ def find_outlier_pixels(frame, tolerance=3, plot_it=False):#worry_about_edges=Tr def correct_images(images, badpixels, verbose=0): ''' + INPUT + ------ + + RESULT + ------ + + NOTE + ------ ''' if type(badpixels) != int: if type(images) == list: @@ -263,8 +368,10 @@ def restore_bad_pixels(frames, bad_pixel, by_list=True, check_neighbours=True, p sure that adjacent pixels are not bad (time consuming). Default is to use a list of bad pixels and a for loop. For many bad pixels consider using the optinal alternative using a bad pixel mask. - IN: - frames - either list of frames as 2D numpy array, + + INPUT + ------ + frames - either list of frames as 2D numpy array, or 3D numpy array (frame number, n_rows, n_cols), or 2D numpy array (n_rows, n_cols) bad_pixel - either list of tuples of bad pixel coordinates, @@ -280,10 +387,13 @@ def restore_bad_pixels(frames, bad_pixel, by_list=True, check_neighbours=True, p results or not (OPTIONAL: if not provided, switched off) verbose - integer of feedback level (amount of prints) - (OPTIONAL: if not provided, only ERROR output) - RETURN: + (OPTIONAL: if not provided, only ERROR output) + RESULT + ------ frames - 3D numpy array (frame number, n_rows, n_cols) of corrected frames + NOTE + ------ """ # make sure frames is correctly shaped @@ -430,6 +540,14 @@ def restore_bad_pixels(frames, bad_pixel, by_list=True, check_neighbours=True, p def generate_new_hot_image(cold,reference_cold,reference_hot): ''' + INPUT + ------ + + RESULT + ------ + + NOTE + ------ ''' if cold is None or reference_cold is None or reference_hot is None: raise Exception("generate_new_hot_image: Cannot Calculate new Hot image, if images are missing!") @@ -438,6 +556,14 @@ def generate_new_hot_image(cold,reference_cold,reference_hot): def calculate_gain_offset_image_pix(cold_image,hot_image=None,reference_cold=None,reference_hot=None,verbose=0): ''' + INPUT + ------ + + RESULT + ------ + + NOTE + ------ ''' if hot_image is None: hot_image=generate_new_hot_image(cold_image,reference_cold,reference_hot) @@ -452,6 +578,16 @@ def calculate_gain_offset_image_pix(cold_image,hot_image=None,reference_cold=Non return Gain_rel,Offset_rel def calculate_gain_offset_image(cold_image,hot_image=None,reference_cold=None,reference_hot=None,verbose=0): + """ + INPUT + ------ + + RESULT + ------ + + NOTE + ------ + """ if hot_image is None: hot_image=generate_new_hot_image(cold_image,reference_cold,reference_hot) if verbose>0: @@ -502,17 +638,53 @@ def calculate_gain_offset_image(cold_image,hot_image=None,reference_cold=None,re #============================================================================== def reconstruct_coldframe (exposuretime, sT, a, bnew, coldref): + """ + INPUT + ------ + exposuretime: integer + the exposure time + sT: + a: + bnew: + coldref: numpy array + the reference cold frame as the base for the reconstruction + RESULT + ------ + + NOTE + ------ + """ cirebuild = a * sT + bnew * exposuretime + coldref return cirebuild #%% other functions def check_dublicates(array): + """ + INPUT + ------ + + RESULT + ------ + + NOTE + ------ + """ a = array import collections return [item for item, count in collections.Counter(a).items() if count > 1] def check_dublicates_2(array): + """ + INPUT + ------ + + RESULT + ------ + + NOTE + ------ + """ seen = set() uniq = [] for x in array: @@ -523,11 +695,29 @@ def check_dublicates_2(array): def get_work_list(pipepath,typ="q"): """ + INPUT + ------ + pipepath: string + the path to the folder where the files are located + typ: string + the typ of data which is requested in the working list\n + possiblities: q, Aw, qpeak, width, load\n + or anything else for the problematic programs + RESULT + ------ + cam_programs: list + a list containing two coloumns, cameras and programs + reasons: list, optional, only for problematic programs + a list showing the reasons, why data are not processed + NOTE + ------ """ today=datetime.datetime.now() cam_programs=[] - if typ in ('q','load'): + if typ in ('q_old','load_old'): f=open(pipepath+str(today.year)+str(today.month)+"_"+typ+"_requests.txt") + elif typ in ('q','load','qpeak','Aw','width'): + f=open(pipepath+"Auto_"+typ+"_requests.txt") else: reasons=[] f = open(pipepath+"problematic_programs.txt") @@ -1191,6 +1381,7 @@ def derive_wetted_area_per_module(heat_flux, mapping, mode='average', q_max=None ------ total_wetted_area: float or numpy array wetted area in a shape that depends on the mode (see NOTES) + in average it is (upper,lower) q_max: float or numpy array peak heat flux used for normalizatin in a shape that depends on the mode (see NOTES) NOTES @@ -1571,4 +1762,44 @@ def derive_wetted_area_per_module(heat_flux, mapping, mode='average', q_max=None plt.legend() plt.show() - return total_wetted_area, q_max \ No newline at end of file + return total_wetted_area, q_max + +def convert_Bad_Pixels(Badpixels,map2D): + """ + transform the badpixel information from the camera images into a bad pixel + information for the heat flux images + + INPUT + ----- + Badpixels: 2D array + image of the bad pixels with 0 no bad pixel and >0 as bad pixel + map2D: dictonary of the + see downloadversionIRdata.download_heatflux_scene_model_reference() + RESULT + ----- + heatflux_badpixel: numpy array + good pixels have a zero, bad pixels have a 1 + """ + heatflux_badpixel=np.zeros(np.shape(map2D['Pixel_Y'])) + PX=map2D['Pixel_X'] + PY=map2D['Pixel_Y'] + + tlocos=np.where(np.asarray(Badpixels)>0) + locos=[] + for i in range(len(tlocos[0])): + locos.append((tlocos[0][i],tlocos[1][i])) + for ele in locos: + Xloc=np.where(PX==ele[1]) + Yloc=np.where(PY==ele[0]) + if len(Xloc[0])>0 and len(Yloc[0])>0:#okay there are points in the heat flux image with the coordinates + PointsX=[] +# PointsY=[] + for k in range(len(Xloc[0])): + PointsX.append((Xloc[0][k],Xloc[1][k])) + for L in range(len(Yloc[0])): + YY=(Yloc[0][L],Yloc[1][L]) + if YY in PointsX: + heatflux_badpixel[YY[0],YY[1]]=1 +# PointsY.append((Yloc[0][L],Yloc[1][L])) + + return heatflux_badpixel \ No newline at end of file diff --git a/ToDO.txt b/ToDO.txt index ccd2421383ddca6edd9d3b785efa1c2fb28a18a3..d03c20de5d2fff827541f84e955a1af730b55179 100644 --- a/ToDO.txt +++ b/ToDO.txt @@ -6,3 +6,6 @@ download: - implement download of the stored temperature data (After the upload) - implement download of the stored heat flux data --> done in V3.0.0 - implement download of FLIR data --> Done in V2.8.0, in testing phase +- implement caching +- implement download of wetted area, peak heat flux and average strike line width +- implement request of all ports diff --git a/downloadversionIRdata.py b/downloadversionIRdata.py index 2938d1a1235a22b7d5e1771893870ff15b2c77f1..be2437c8cefd5c2a849d882842cef992348140df 100644 --- a/downloadversionIRdata.py +++ b/downloadversionIRdata.py @@ -4,7 +4,7 @@ Created on Wed Oct 25 15:51:46 2017 updated on Tue Aug 21 10:20:00 2018 last update on Fr Nov 23 15:37:00 2018 -Version: 3.4.0 +Version: 3.4.1 (Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities @@ -12,7 +12,7 @@ Version: 3.4.0 #number of updates for bug fixes ) @author: holn """ -version="V3.4.0" +version = "V3.4.1" import numpy as np import IR_image_tools as IR_tools @@ -28,7 +28,7 @@ from IR_config_constants import archivepath, testarchivepath, portcamdict, camli import datetime import urllib.request import urllib.error - +import logging import json from PIL import Image from io import BytesIO @@ -36,7 +36,7 @@ from os.path import join import matplotlib.pyplot as plt try: import threading - import archivedb as AKF_2 + import w7xarchive as AKF_2 # disbale deprecation warnings, because of timezone warning spam import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) @@ -81,12 +81,12 @@ def get_latest_version(stream, project="W7X", testmode=False, t_from=None, return int of version number of None if non-versioned or non-existing stream author: G. Schlisio, edit by holn """ - if testmode: + if testmode:#check which archive should be used base = testarchivepath else: base = archivepath - if program == None and t_from == None and t_to == None: + if program == None and t_from == None and t_to == None:# check wheather any time information was given request = urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"}) else: if program is not None: @@ -124,19 +124,44 @@ def get_latest_version(stream, project="W7X", testmode=False, t_from=None, def TimeToNs(date, time): """ - TimeToNs(date, time) - (list) date [year, month, day] - (list) time [hours, minutes, seconds, microseconds] + TimeToNs(date, time) + + INPUT + ------ + data: list + [year, month, day] + time: list + [hours, minutes, seconds, microseconds] + RESULT + ------ + nsdate: integer + nanosecond since 1.1.1970 0:00:00 """ date_time = datetime.datetime(date[0], date[1], date[2], time[0], time[1], time[2], time[3]) div = date_time-datetime.datetime(1970, 1, 1, 0, 0, 0) nsdate = div.total_seconds()*1e9 - return nsdate + return int(nsdate) def read_program(timestamp_start, timestamp_end=0, tol=60): """ - read_program() + read_program() downloads the information from the W7-X ArchiveDB and provides the programnumber and triggers + + INPUT + ------ + timestamp_start: integer + start time in nanoseconds + timestamp_end: inter, optional, default is 0 + time to stop the search in nanoseconds + tol: integer, optional, default is 60 + toleranz in seconds around the timestamp_start for searching the program in the W7-X database + RESULT + ------ + exist: boolean + True if data was found, False if not + program: dictionary + W7-X program information + """ program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from' if timestamp_end == 0: @@ -167,26 +192,56 @@ def download_LUT(port, time, exposure=0, emissivity=0, camera_filter=0, version= testmode=False, verbose=0): """ download_LUT(camera, port, time, exposure=0, emissivity=0, camera_filter=0, version=1): - time in ns + download of the look up table for the infrared cameras from the database for OP1.2(a+b) Have to swap 11, 21 until correction in the database + INPUT + ------ + port: interger + number of the AEF camera port + time: integer + nanosecond timestamp for the LUT + exposure: integer + requested exposure time in microseconds + emissivity: float, string or integer + emissivity value to identify which LUT is requested + camera_filter: inter or string, only needed for port 50 + filter number of the Infratec camera in port AEF50 + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if Look up table was found, False if not + LUT: list + contains the calibration information: [digital level, temperature, temperature error] + NOTE + ------ + function need to be adapted for OP2, if AEK ports come into opperation """ - OP = IR_tools.get_OP_by_time(time_ns=time) - if port == 21: + OP = IR_tools.get_OP_by_time(time_ns=time)#getting operation phase + if port == 21:# swapping both ports (21 and 11) port = 11 elif port == 11: port = 21 - if testmode: + if testmode: #check which archive should be used larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_" else: larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_" + #camera dependend request string generation if port == 50 and OP == "OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec": query = "Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity)) elif port in [10, 11, 20, 21, 30, 31, 40, 41, 51] or OP == "OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam": query = "Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity)) else: - print("download_LUT: Error! Camera unknown, stopping here.") - raise Exception - if version == 0: +# print("download_LUT: Error! Camera unknown, stopping here.") + logging.warning("download_LUT: Error! Camera unknown, stopping here.") + return False, 0 +# raise Exception + if version == 0:#version check, if 0 version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM", t_from=time, testmode=testmode) if verbose > 0: print("download_LUT: LUT V"+str(version)+" is used") @@ -216,6 +271,29 @@ def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, verbose=0):# read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1) Read LUT V3 from local files for testing new calibration. + INPUT + ------ + port: integer or string + the port number of the AEF port + this_time: integer + nanosecond timestamp for the LUT + t_exp: integer + requested exposure time in microseconds + emissivity: float, string or integer + emissivity value to identify which LUT is requested + cfilter: inter or string, only needed for port 50 + filter number of the Infratec camera in port AEF50 + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if Look up table was found, False if not + LUT: list + contains the calibration information: [digital level, temperature, temperature error] + NOTE + ------ + function need to be adapted for OP2, if AEK ports come into opperation """ LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT' OP = IR_tools.get_OP_by_time(time_ns=this_time) @@ -235,6 +313,32 @@ def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, verbose=0):# def download_NUC_by_program(port, program, exposure, version=0, testmode=False, verbose=0): ''' + download the NUC data for a given port, program and exposure time of the camera + INPUT + ------ + port: integer or string + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + exposure: integer + exposure time in microseconds + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if NUC data was found, False if not + NUC: list + a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error + NUC_describtion: list + a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error'] + NOTE + ------ + function need to be adapted for OP2, if AEK ports come into opperation ''' # prog=get_program_from_PID(program) # try: @@ -253,6 +357,34 @@ def download_NUC_by_program(port, program, exposure, version=0, testmode=False, def download_NUC_by_times(port, starttime, stoptime, exposure, testmode=False, version=0, verbose=0): ''' + download the NUC data for a given port, start and endtime in ns and exposure time of the camera + INPUT + ------ + port: integer or string + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + exposure: integer + exposure time in microseconds + version: integer, optional + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional + if True, the data will be loaded from the test archive + verbose: integer, optional + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if NUC data was found, False if not + NUC: list + a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error + NUC_describtion: list + a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error'] + NOTE + ------ + function need to be adapted for OP2, if AEK ports come into opperation ''' OP = IR_tools.get_OP_by_time(time_ns=starttime) if testmode: @@ -303,6 +435,31 @@ def get_NUC_by_program(port, program, exposure, version=0, testmode=False, verbo Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera in a given port and time interval. Depending on time and camera, reconstruct the cold frame and/or compute the gain, offset from the hot and cold frame. + INPUT + ------ + port: integer or string + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + exposure: integer + exposure time in microseconds + version: integer, optional + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional + if True, the data will be loaded from the test archive + verbose: integer, optional + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if NUC data was found, False if not + NUC: list + a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error + NUC_describtion: list + a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error'] + NOTE + ------ + function need to be adapted for OP2, if AEK ports come into opperation ''' # prog=get_program_from_PID(program) # try: @@ -326,15 +483,46 @@ def get_NUC_by_times(port, starttime, stoptime, t_exp, version=0, testmode=False Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera in a given port and time interval. Depending on time and camera, reconstruct the cold frame and/or compute the gain, offset from the hot and cold frame. + the result is operation phase dependent, due to changes in the aquisition and saving programs + + INPUT + ------ + port: integer or string + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + exposure: integer + exposure time in microseconds + version: integer, optional + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional + if True, the data will be loaded from the test archive + OP: string, optional, default None + name of operation phase + verbose: integer, optional + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if NUC data was found, False if not + NUC: list + a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error + NUC_describtion: list + a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error'] + NOTE + ------ + function need to be adapted for OP2, if AEK ports come into opperation """ - if OP is None: + if OP is None: #get operation phase if not given OP = IR_tools.get_OP_by_time(time_ns=stoptime) - if OP == "OP1.2a": + if OP == "OP1.2a": # check for operation phase t1 = stoptime t0 = starttime prog = read_program(t1) program = prog[1]['id'] - if (port == 31) or (port == 21 and float(program[4:]) > 1110): + if (port == 31) or (port == 21 and float(program[4:]) > 1110):#camera 31 and 21 need after 20171110 new coldframes due to a open shutter during the recording of the coldframes if verbose > 0: print("get_NUC_by_times: rebuilding coldframe") #use any way the rebuilt coldframe. @@ -346,7 +534,9 @@ def get_NUC_by_times(port, starttime, stoptime, t_exp, version=0, testmode=False coldref = hotcold[1] hotref = hotcold[0] else: - raise Exception("get_NUC_by_times: unable to download reference frames") +# raise Exception("get_NUC_by_times: unable to download reference frames") + logging.warning("get_NUC_by_times: unable to download reference frames") + return False, 0, 0 filestring = 'AEF' + str(port) + '_et' + str(int(t_exp)) amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy')) bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy')) @@ -360,7 +550,7 @@ def get_NUC_by_times(port, starttime, stoptime, t_exp, version=0, testmode=False gain_error = 0 offset_error = 0 return True, [gain, offset, cirebuild, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error'] - elif (port == 11) or (port == 21): + elif (port == 11) or (port == 21): # case of the swaped cameras if verbose > 0: print('get_NUC_by_times: downloading NUC') # coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp) @@ -369,10 +559,14 @@ def get_NUC_by_times(port, starttime, stoptime, t_exp, version=0, testmode=False coldref = hotcold[1] hotref = hotcold[0] else: - raise Exception("get_NUC_by_times: unable to download reference frames") +# raise Exception("get_NUC_by_times: unable to download reference frames") + logging.warning("get_NUC_by_times: unable to download reference frames") + return False, 0, 0 NUC_DL = download_NUC_by_times(port, t0, t1, t_exp, version, testmode) if NUC_DL[0] == False: - raise Exception("get_NUC_by_times: NUC was not found") +# raise Exception("get_NUC_by_times: NUC was not found") + logging.warning("get_NUC_by_times: NUC was not found") + return False, 0, 0 else: ci = NUC_DL[1][2] gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref, verbose=verbose-1) @@ -382,7 +576,7 @@ def get_NUC_by_times(port, starttime, stoptime, t_exp, version=0, testmode=False gain_error = 0 offset_error = 0 return True, [gain, offset, ci, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error'] - else: + else: #all other cases, the "normal" case if verbose > 0: print('get_NUC_by_times: downloading NUC') NUC_DL = download_NUC_by_times(port, starttime, stoptime, t_exp, testmode=testmode) @@ -399,13 +593,15 @@ def get_NUC_by_times(port, starttime, stoptime, t_exp, version=0, testmode=False coldref = hotcold[1] hotref = hotcold[0] else: - raise Exception("get_NUC_by_times: unable to download reference frames") +# raise Exception("get_NUC_by_times: unable to download reference frames") + logging.warning("get_NUC_by_times: unable to download reference frames") + return False, 0, 0 ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way try: exist, _, coldframes = download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, frametype=0, version=0, testmode=testmode, verbose=verbose-1) except: exist = False - if exist: + if exist:#okay it exist, average the cold frames cold = np.zeros(np.shape(coldframes[0]), dtype=np.uint64) for ele in coldframes: cold += ele @@ -418,22 +614,54 @@ def get_NUC_by_times(port, starttime, stoptime, t_exp, version=0, testmode=False badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1) return True, [gain, offset, cold, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error'] else: - raise Exception("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP") +# raise Exception("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP") + logging.warning("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP") + return False, 0, 0 def download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, frametype=0, version=0, testmode=False, verbose=0): """ - download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, frametype=0, version=0) + download of the calibration raw files which are recorded between t0 and t1 trigger frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background) + + INPUT + ------ + port: integer + port number of the AEF port of the camera + t_exp: integer + exposure time in microseconds + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + frametype: integer, default 0 + defines which type of frames should be returned, 0 for cold frames, 1 for background frames + version: integer, optional + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional + if True, the data will be loaded from the test archive + verbose: integer, optional + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + NOTE + ------ + function need to be adapted for OP2, if AEK ports come into opperation """ gotit, time_t, texp_t = get_exposure_by_times(port, int(starttime-100), int(stoptime-20e6), testmode=testmode) OP = IR_tools.get_OP_by_time(time_ns=stoptime) if gotit: expinds = np.where(texp_t == t_exp)[0] if len(expinds) == 0: - print("download_calibration_raw_files_by_time: Error! cannot find the exposure time in the given data") + logging.warning("download_calibration_raw_files_by_time: Error! cannot find the exposure time in the given data") return False, [0], [0] else: - print("download_calibration_raw_files_by_time: Error! exposure time not found") + logging.warning("download_calibration_raw_files_by_time: Error! exposure time not found") return False, [0], [0] gotitf, timef, values_f = get_frametype_by_times(port, int(starttime-100), int(stoptime-20e6), testmode=testmode) if gotitf: @@ -456,7 +684,9 @@ def download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, fra ref_t = [np.min(frametimes[turnpoint+1:]), np.max(frametimes[turnpoint+1:])] # print((ref_t[1]-ref_t[0])/1e9) else: - raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!") +# raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!") + logging.warning("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!") + return False, [0], [0] else: print("download_calibration_raw_files_by_time: Error! frametype not found") frametimes = time_t[expinds] @@ -470,7 +700,9 @@ def download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, fra ref_t = [np.min(frametimes[turnpoint+1:]), np.max(frametimes[turnpoint+1:])] print((ref_t[1]-ref_t[0])/1e9) else: - raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!") +# raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!") + logging.warning("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!") + return False, [0], [0] # return False, [0], [0] t1date = datetime.datetime.utcfromtimestamp((stoptime-100)/1e9) t1date = t1date.isoformat() @@ -497,6 +729,31 @@ def download_hot_cold_reference_by_times(port, exposure, starttime=1503907200000 """ Loads the most recent hot and cold calibration frames for a starttime. Uses first calibration frames if time is not defined. + INPUT + ------ + port: integer or string + port number of the AEF port of the camera, or string containing the full port name + exposure: integer + exposure time in microseconds + starttime: integer + time in ns where the search should start + version: integer, optional + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional + if True, the data will be loaded from the test archive + verbose: integer, optional + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + images: list + a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error + image_describtion: list + a list which entry in NUC is what, looks like ['hot','cold'] + NOTE + ------ + function need to be adapted for OP2, if AEK ports come into opperation """ OP = IR_tools.get_OP_by_time(time_ns=starttime) if isinstance(port, int): @@ -551,7 +808,37 @@ def download_hot_cold_reference_by_times(port, exposure, starttime=1503907200000 def download_background_by_program(port, program, exposure, camera_filter=0, version=0, testmode=False, verbose=0): ''' + download the background from the AEF cameras in OP1.2a, which are recorded between t0 and t1 trigger returned exist, time, frame + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + exposure: integer + exposure time in microseconds + camera_filter: integer or string, only needed for port 50 + filter number of the Infratec camera in port AEF50 + version: integer, optional + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional + if True, the data will be loaded from the test archive + verbose: integer, optional + feedback level (details of print messages) + OP: string, optional, default None + name of operation phase + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + NOTE + ------ + function need to be adapted for OP2, if AEK ports come into opperation ''' # prog=get_program_from_PID(program) # try: @@ -569,6 +856,40 @@ def download_background_by_program(port, program, exposure, camera_filter=0, ver def download_background_by_times(port, starttime, stoptime, exposure, camera_filter=0, version=0, testmode=False, verbose=0): + """ + download the background frames for a given time intervall in nanoseconds, defined by start and stop time + + INPUT + ------ + port: integer + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + exposure: integer + exposure time in microseconds + camera_filter: integer or string, only needed for port 50 + filter number of the Infratec camera in port AEF50 + version: integer, optional + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional + if True, the data will be loaded from the test archive + verbose: integer, optional + feedback level (details of print messages) + + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + NOTE + ------ + function need to be adapted for OP2, if AEK ports come into opperation + """ OP = IR_tools.get_OP_by_time(time_ns=starttime) if OP == "OP1.2a": stream = portpathdict[OP]["AEF"+str(port)]+"background_" @@ -615,8 +936,9 @@ def download_background_by_times(port, starttime, stoptime, exposure, camera_fil elif OP == "OP1.2b": NUC_DL = get_NUC_by_times(port, starttime, stoptime, t_exp=exposure, version=version, testmode=testmode, verbose=verbose-1)#download_NUC_by_times(port, t0, t1, t_exp, version) if NUC_DL[0] == False: - raise Warning("download_background_by_times: NUC was not found") -# return False, 0, 0 +# raise Warning("download_background_by_times: NUC was not found") + logging.warning("download_background_by_times: NUC was not found") + return False, 0, 0 else: # extract NUC elements gain = [] @@ -632,6 +954,42 @@ def download_background_by_times(port, starttime, stoptime, exposure, camera_fil def get_NUCed_background_by_times(port, t0, t1, t_exp, cfilter, gain, offset, version=0, testmode=False, plot_it=False, verbose=0): """ OP1.2b function + download the background frame and applies the NUC onto the background, given by gain und offset + it is an average over 30 frames. + + INPUT + ------ + port: integer + port number of the AEF port of the camera + t0: integer + nanoseconds timestamp of the t0 trigger, start of the program + t1: integer + nanoseconds timestamp of the t1 trigger, start of the heating + t_exp: integer + exposure time in microseconds + gain: numpy array + the gain image from the calibration to convert the counts into digital level + offset: numpy array + the offset image from the calibration to convert the counts into digital level + version: integer, optional + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional + if True, the data will be loaded from the test archive + plot_it: boolean, optional, default False + if True, the data will be plotted directly + verbose: integer, optional + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: integer + the timestamp of the background frame (firest timestamp of the 30 frames) + background_frame: numpy array + the background frame + NOTE + ------ + function need to be adapted for OP2, if AEK ports come into opperation """ exist, btime, backgroundframes = download_calibration_raw_files_by_time(port, t_exp, t0, t1, frametype=1, version=version, testmode=testmode, verbose=verbose-1) camera = portcamdict["OP1.2b"]["AEF"+str(port)] @@ -743,6 +1101,35 @@ def get_NUCed_coldframe_by_program(port, program, exposure=None, version=0, def download_raw_images_by_program(port, program, time_window=0, version=0, testmode=False, verbose=0): """ + download the raw frames (counts) from the infrared cameras for a given program + + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + time_window: list, array, integer or float, optional, default 0 + if integer or float: size of the window in seconds from t1 + if list or array: start and endpoint of the window in seconds from t1 + if 0 the full program will be downloaded (t1 until t6) + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + NOTE + ------ + the timewindow will be substructured into intervalls, and each intervall will be downloaded one after the other to avoid a overfilling of the RAM. A to large intervall can still overflow the RAM """ exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) # prog=AKF_1.get_program_from_PID(program) @@ -755,11 +1142,15 @@ def download_raw_images_by_program(port, program, time_window=0, version=0, test if exist: if type(time_window) == list or type(time_window) == np.ndarray: if len(time_window) > 2 or time_window[0] > 1000: - raise Exception("download_raw_images_by_program: invalid input for the time_window!") +# raise Exception("download_raw_images_by_program: invalid input for the time_window!") + logging.warning("download_raw_images_by_program: invalid input for the time_window!") + return False, 0, 0 tstart = time_window[0] tstop = time_window[1] if tstop <tstart: - raise Exception("download_raw_images_by_program: t_stop before t_start") +# raise Exception("download_raw_images_by_program: t_stop before t_start") + logging.warning("download_raw_images_by_program: t_stop before t_start") + return False, 0, 0 return download_raw_images_by_times(port, int(starttime+tstart*1e9), int(starttime+tstop*1e9), version, testmode=testmode, verbose=verbose) else: if time_window == 0: @@ -772,6 +1163,35 @@ def download_raw_images_by_program(port, program, time_window=0, version=0, test def download_raw_images_by_times(port, starttime, stoptime, version=0, intervalSize=1E9, testmode=False, verbose=0): """ + download the raw frames (counts) from the infrared cameras for a given time interval in nanoseconds + + INPUT + ------ + port: integer + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + intervalSize: integer, optional, default 1e9 (1 second) + size of the intervall to be downloaded in nanoseconds + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + NOTE + ------ + the timewindow will be substructured into intervalls, and each intervall will be downloaded one after the other to avoid a overfilling of the RAM. A to large intervall can still overflow the RAM """ OP = IR_tools.get_OP_by_time(time_ns=starttime) if version == 0: @@ -784,6 +1204,29 @@ def download_raw_images_by_times(port, starttime, stoptime, version=0, intervalS def download_images_by_times(larchivepath, starttime, stoptime, intervalSize=1E9, verbose=0):#version=0, """ + INPUT + ------ + larchivepath: string + local archivepath, the URL to download the images + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + intervalSize: integer, optional, default 1e9 (1 second) + size of the intervall to be downloaded in nanoseconds + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + NOTE + ------ + the timewindow will be substructured into intervalls, and each intervall will be downloaded one after the other to avoid a overfilling of the RAM. A to large intervall can still overflow the RAM """ if '_raw' in larchivepath: typo = np.uint16 @@ -848,6 +1291,38 @@ if fastDL: def download_raw_images_by_program_via_png(port, program, time_window=0, version=0, threads=1, testmode=False, verbose=0): """ + download of the raw images in png format instead of json for a given program. + number of threads can be defined to download several pngs in parallel + + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + time_window: list, array, integer or float, optional, default 0 + if integer or float: size of the window in seconds from t1 + if list or array: start and endpoint of the window in seconds from t1 + if 0 the full program will be downloaded (t1 until t6) + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + threads: integer, default 1 + number of parallel threads to download the images + (approx. max. 2x number of cores and max. timeintervall/0.5 for best performance) + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: numpa array + an array containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + NOTE + ------ """ # prog=get_program_from_PID(program) ## try: @@ -880,16 +1355,54 @@ if fastDL: def download_images_by_time_via_png(larchivepath, starttime, stoptime, time_window=0, threads=1, version=0, verbose=0, framerate='max'): """ + download of the images in png format instead of json for a given timeintervall defined by start and stop time. + number of threads can be defined to download several pngs in parallel + + INPUT + ------ + larchivepath: string + local archivepath, the URL to download the images + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + time_window: list, array, integer or float, optional, default 0 + if integer or float: size of the window in seconds from t1 + if list or array: start and endpoint of the window in seconds from t1 + if 0 the full program will be downloaded (t1 until t6) + threads: integer, default 1 + number of parallel threads to download the images + (approx. max. 2x number of cores and max. timeintervall/0.5 for best performance) + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + verbose: integer, optional, default 0 + feedback level (details of print messages) + framerate: string or integer, optional, default 'max' + sets the framerate to downsample the data. normal rate is 100. + RESULT + ------ + exist: boolean + True if data was found, False if not + time: numpa array + an array containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + NOTE + ------ """ success = True if type(time_window) == list: tstart = time_window[0] tstop = time_window[1] if tstop < tstart: - raise Exception("download_images_by_time_via_png: t_stop before t_start") +# raise Exception("download_images_by_time_via_png: t_stop before t_start") + logging.warning("download_images_by_time_via_png: t_stop before t_start") + return False, [0], [0] time_from = (starttime-100)+int(tstart*1E9) if stoptime < time_from: - raise Exception("download_images_by_time_via_png: time_window after stoptime") +# raise Exception("download_images_by_time_via_png: time_window after stoptime") + logging.warning("download_images_by_time_via_png: time_window after stoptime") + return False, [0], [0] time_to = starttime+int(tstop*1E9) if stoptime < time_to: time_to = stoptime @@ -917,8 +1430,9 @@ if fastDL: except Exception as E: # print('download_images_by_time_via_png: Error loading times from ', larchivepath, stdate.replace("T", " "), enddate.replace("T", " ")) print('download_images_by_time_via_png: Error loading times from {0} between [{1}, {2}]'.format(larchivepath, time_from, time_to)) - raise Warning(E) -# return False, [0], [0] +# raise Warning(E) + logging.warning(E) + return False, [0], [0] if framerate != 'max': if verbose > 5: @@ -983,7 +1497,31 @@ if fastDL: class download_images_thread(threading.Thread): + """ + class for parallel download of images.\n + contains an init function and a run function + """ def __init__(self, threadID, larchivepath, times, resultlist, version=0): + """ + INPUT + ------ + theadID: integer + identification number of the thread + larchivepath: string + local archivepath, the URL to download the images + times: list + the timestamps of each frames for the download + resultlist: list + list which is returned after the thread is done + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + RESULT + ------ + resultlist: list + list containing the ID, exist, time and the images + NOTE + ------ + """ threading.Thread.__init__(self) self.threadID = threadID self.times = times @@ -1005,9 +1543,55 @@ if fastDL: self.resultlist.append([self.threadID, successes, time, images]) class convert_raw_to_temp_thread(threading.Thread): + """ + class for parallel download of images and parallel application the NUC and temperature conversion.\n + contains an init function and a run function + """ def __init__(self, larchivepath, times,resultlist, threadID, version=0, background=0, LUT=[[], [], []],refT=28.5, gain=0, offset=0, gain_error=0, offset_error=0, give_ERROR=False, FOV=0, badpixels=[], verbose=0): + """ + INPUT + ------ + larchivepath: string + local archivepath, the URL to download the images + times: list + the timestamps of each frames for the download + resultlist: list + list which is returned after the thread is done + theadID: integer + identification number of the thread + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + background: numpy array + background image + LUT: list + the look up table, containing the digital level, the temperature and the temperature error + refT: float + reference temperature for the calibration + gain: numpy array + the gain image from the calibration to convert the counts into digital level + offset: numpy array + the offset image from the calibration to convert the counts into digital level + gain_error: numpy array, optional + the gain error image from the calibration to calculate the error, needed if give_ERROR=True + offset_error: numpy array, optional + the offset error image from the calibration to calculate the error, needed if give_ERROR=True + give_ERROR: boolean + switch to activate the error calculation and returning of the error + FOV: numpy array + field of view image, with 1 in the visible region and 0 in the dark region + badpixels: list + list of the bad pixels for the corrections + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + resultlist: list + list containing the ID, exist, time and the images + NOTE + ------ + """ threading.Thread.__init__(self) self.threadID = threadID self.times = times @@ -1026,6 +1610,17 @@ if fastDL: self.badpixels = badpixels self.verbose = verbose def run(self): + """ + INPUT + ------ + same as init + RESULT + ------ + resultlist: list + list containing the ID, exist, time and the images + NOTE + ------ + """ images = [] time = [] successes = [] @@ -1050,9 +1645,54 @@ if fastDL: self.resultlist.append([self.threadID, successes, time, images]) class NUC_raw_thread(threading.Thread): + """ + class for parallel calculation of nuced raw images + """ def __init__ (self, larchivepath, times,resultlist, threadID, version=0, background=0, LUT=[[], [], []],refT=28.5, gain=0, offset=0, gain_error=0, offset_error=0, give_ERROR=False, FOV=0, badpixels=[], verbose=0): + """ + INPUT + ------ + larchivepath: string + local archivepath, the URL to download the images + times: list + the timestamps of each frames for the download + resultlist: list + list which is returned after the thread is done + theadID: integer + identification number of the thread + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + background: numpy array + background image + LUT: list + the look up table, containing the digital level, the temperature and the temperature error + refT: float + reference temperature for the calibration + gain: numpy array + the gain image from the calibration to convert the counts into digital level + offset: numpy array + the offset image from the calibration to convert the counts into digital level + gain_error: numpy array, optional + the gain error image from the calibration to calculate the error, needed if give_ERROR=True + offset_error: numpy array, optional + the offset error image from the calibration to calculate the error, needed if give_ERROR=True + give_ERROR: boolean + switch to activate the error calculation and returning of the error + FOV: numpy array + field of view image, with 1 in the visible region and 0 in the dark region + badpixels: list + list of the bad pixels for the corrections + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + resultlist: list + list containing the ID, exist, time and the images + NOTE + ------ + """ threading.Thread.__init__(self) self.threadID = threadID self.times = times @@ -1071,6 +1711,9 @@ if fastDL: self.badpixels=badpixels self.verbose=verbose def run(self): + """ + + """ images=[] time=[] successes=[] @@ -1098,6 +1741,42 @@ if fastDL: def download_raw_FLIR_images_via_png(t1, t6, time_window=0, version=0, threads=1, verbose=0, testmode=True): """ + INPUT + ------ + t1: integer + nanoseconds timestamp of the t1 trigger, start of the heating + t6: integer + nanoseconds timestamp of the t6 trigger, end of the program + time_window: list, array, integer or float, optional, default 0 + if integer or float: size of the window in seconds from t1 + if list or array: start and endpoint of the window in seconds from t1 + if 0 the full program will be downloaded (t1 until t6) + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + threads: integer, default 1 + number of parallel threads to download the images + (approx. max. 2x number of cores and max. timeintervall/0.5 for best performance) + verbose: integer, optional, default 0 + feedback level (details of print messages) + testmode: boolean, optional, default True + if True, the data will be loaded from the test archive + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + mode: integer + kind of data structure,\n 1= pre-sets stored in different lists\n; 2=presets stored interlaced in one list + tsets: list + contains the information which frame belongs to which pre-set + versions: list + contains the informations of the version of each pre-set + NOTE + ------ + the FLIR camera is capable to run in superframing mode, recording frames with different exposure times in an interlaced mode. each exposure time frame is stored in a different file named after the pre-set """ if testmode: larchivepath=testarchivepath+project+"/QSR07_FLIR/AEK51_raw" @@ -1107,7 +1786,9 @@ def download_raw_FLIR_images_via_png(t1, t6, time_window=0, version=0, threads=1 tstart=time_window[0] tstop=time_window[1] if tstop<tstart: - raise Exception("download_raw_FLIR_images_via_png: t_stop before t_start") +# raise Exception("download_raw_FLIR_images_via_png: t_stop before t_start") + logging.warning("download_raw_FLIR_images_via_png: t_stop before t_start") + return False, [0], [0], 0, [0], [0] starttime=int(t1-100+tstart*1e9) stoptime=int(t1+tstop*1e9) # stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart) @@ -1143,7 +1824,9 @@ def download_raw_FLIR_images_via_png(t1, t6, time_window=0, version=0, threads=1 ### we not know the times for all presets for the program, now we have to download them, if times is not empty exist, time, frames, tsets=download_FLIR_Psets(larchivepath, timess, threads, versions, verbose=verbose-1) elif mode==2:##the data is stored only in one datastream, at least this datastream exist - raise Exception("download_raw_FLIR_images_via_png: not implemented") +# raise Exception("download_raw_FLIR_images_via_png: not implemented") + logging.warning("download_raw_FLIR_images_via_png: not implemented") + return False, [0], [0], 0, [0], [0] #download data and downlaod LUT # time=[] @@ -1160,12 +1843,36 @@ def download_raw_FLIR_images_via_png(t1, t6, time_window=0, version=0, threads=1 # success=False # return success, np.array(time), images else: - raise Exception("download_raw_FLIR_images_via_png: cannot identifie the way the searched data is stored!") +# raise Exception("download_raw_FLIR_images_via_png: cannot identifie the way the searched data is stored!") + logging.warning("download_raw_FLIR_images_via_png: cannot identifie the way the searched data is stored!") + return False, [0], [0], 0, [0], [0] return exist, time, frames,mode, tsets, versions def get_FLIR_Pset_times(stdate, enddate, version=0, verbose=0, testmode=True): """ + Download of the timevector for the different pre-sets of the recorded FLIR images + + INPUT + ------ + stdate: integer + time in ns where the search should start + enddate: integer + time in sn where the search should end + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + verbose: integer, optional, default 0 + feedback level (details of print messages) + testmode: boolean, optional, default True + if True, the data will be loaded from the test archive + RESULT + ------ + timess: list + timevectors of each pre-set + versions: list + list of the versions of each pre-set + NOTE + ------ """ if testmode: larchivepath=testarchivepath+project+"/QSR07_FLIR/AEK51_raw" @@ -1222,6 +1929,32 @@ def get_FLIR_Pset_times(stdate, enddate, version=0, verbose=0, testmode=True): def download_FLIR_Psets(larchivepath, timess, threads, versions, verbose=0): """ + Download of the FLIR datasets, which are stored as different datasets. Each dataset has a different exposure time + + INPUT + ------ + larchivepath: string + local archivepath, the URL to download the images + timess: list + list of the timevectors for each pre-set (pset) + threads: integer, default 1 + number of parallel threads to download the images, one or two per pre-set + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + tsets: list + contains the information which frames belong to which pre-set + NOTE + ------ """ if len(timess[0])==len(timess[1]) and len(timess[2])==len(timess[3]) and len(timess[0])==0: exist=False @@ -1323,6 +2056,24 @@ def download_FLIR_Psets(larchivepath, timess, threads, versions, verbose=0): def download_last_raw_image_by_time(larchivepath, starttime, stoptime):#, version=0 """ + downloads for a given time intervall the image as png. it if always the last image in the intervall + + INPUT + ------ + larchivepath: string + local archivepath, the URL to download the images + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + RESULT + ------ + exist: boolean + True if data was found, False if not + frame: list + a list of frame, 2D numpy array + NOTE + ------ """ try: url='{0}/_signal.png?from={1}&upto={2}'.format(larchivepath, starttime-10, stoptime) @@ -1341,6 +2092,31 @@ def download_last_raw_image_by_time(larchivepath, starttime, stoptime):#, versio def download_raw_parlog_by_program(port, program, version=0, testmode=False, verbose=0): """ + download of the additional information for the raw data of the AEF cameras for a given program, which are stored in the parlog. contains information as frame width, height bitdepth. + + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of the parlog (from to) + meta-data: dictionary/list + the meta-data from the parlog converted from JSON + NOTE + ------ """ ## prog=AKF_1.get_program_from_PID(program) # try: @@ -1358,6 +2134,33 @@ def download_raw_parlog_by_program(port, program, version=0, testmode=False, ver def download_raw_parlog_by_times(port, starttime, stoptime, version=0, testmode=False): """ + download of the additional information for the raw data of the AEF cameras for a given time intervall, which are stored in the parlog. contains information as frame width, height bitdepth. + + INPUT + ------ + port: integer + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of the parlog (from to) + meta-data: dictionary/list + the meta-data from the parlog converted from JSON + NOTE + ------ """ OP = IR_tools.get_OP_by_time(time_ns=stoptime) if version == 0: @@ -1385,6 +2188,30 @@ def download_raw_parlog_by_times(port, starttime, stoptime, version=0, testmode= def get_INFRATEC_filter_by_program(program, port=50, version=0, testmode=False, verbose=0): """ + downlods the information of the used filter of the INFRATEC camera for a given program + + INPUT + ------ + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + port: integer, optional, default 50 + port number of the AEF port of the camera + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of the parlog (from to) + filter: integer + the number of the used filter + NOTE + ------ """ # prog=get_program_from_PID(program) ## try: @@ -1402,6 +2229,32 @@ def get_INFRATEC_filter_by_program(program, port=50, version=0, testmode=False, def get_INFRATEC_filter_by_times(starttime, stoptime, port=50, version=0, testmode=False): """ + Idownlods the information of the used filter of the INFRATEC camera for a given time intervall + + INPUT + ------ + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + port: integer, optional, default 50 + port number of the AEF port of the camera + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of the parlog (from to) + filter: integer + the number of the used filter + NOTE + ------ """ OP = IR_tools.get_OP_by_time(time_ns=starttime) if version == 0: @@ -1429,6 +2282,31 @@ def get_INFRATEC_filter_by_times(starttime, stoptime, port=50, version=0, testmo def get_exposure_by_program(port, program, version=0, testmode=False, verbose=0): """ + download the exposure time for a given camera and program for each frame + + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + exposure time: list + the exposure time of the camera for each frame + NOTE + ------ """ # prog=get_program_from_PID(program) ## try: @@ -1447,6 +2325,30 @@ def get_exposure_by_program(port, program, version=0, testmode=False, verbose=0) def get_exposure_by_times(port, starttime, stoptime, version=0, testmode=False): """ + download the exposure time for a given camera and each frame in a given time intervall + + INPUT + ------ + port: integer + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + exposure time: list + the exposure time of the camera for each frame + NOTE + ------ """ OP = IR_tools.get_OP_by_time(time_ns=stoptime) if testmode: @@ -1460,6 +2362,31 @@ def get_exposure_by_times(port, starttime, stoptime, version=0, testmode=False): def get_camera_temp_by_program(port, program, version=0, testmode=False, verbose=0): """ + download the camera temperature for a given camera and program for each frame + + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + temperature: list + the temperature of the camera for each frame + NOTE + ------ """ ## prog=AKF_1.get_program_from_PID(program) # try: @@ -1477,6 +2404,30 @@ def get_camera_temp_by_program(port, program, version=0, testmode=False, verbose def get_camera_temp_by_times(port, starttime, stoptime, version=0, testmode=False): """ + download the camera temperature for a given camera and each frame in a given time intervall + + INPUT + ------ + port: integer + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + temperature: list + the temperature of the camera for each frame + NOTE + ----- """ OP = IR_tools.get_OP_by_time(time_ns=stoptime) stream=portpathdict[OP]["AEF"+str(port)]+"meta_" @@ -1494,6 +2445,31 @@ def get_camera_temp_by_times(port, starttime, stoptime, version=0, testmode=Fals def get_sensor_temp_by_program(port, program, version=0, testmode=False, verbose=0): """ + download the sensor temperature for a given camera and program for each frame + + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + temperature: list + the temperature of the camera sensor for each frame + NOTE + ------ """ # prog=get_program_from_PID(program) ## try: @@ -1511,6 +2487,30 @@ def get_sensor_temp_by_program(port, program, version=0, testmode=False, verbose def get_sensor_temp_by_times(port, starttime, stoptime, version=0, testmode=False): """ + download the sensor temperature for a given camera and each frame in a given time intervall + + INPUT + ------ + port: integer + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + temperature: list + the temperature of the camera sensor for each frame + NOTE + ----- """ OP = IR_tools.get_OP_by_time(time_ns=stoptime) stream=portpathdict[OP]["AEF"+str(port)]+"meta_" @@ -1524,6 +2524,31 @@ def get_sensor_temp_by_times(port, starttime, stoptime, version=0, testmode=Fals def get_camera_framerate_by_program(port, program, version=0, testmode=False, verbose=0): """ + download the framerate for a given camera and program for each frame + + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + framerate: list + the framerate of the camera for each frame + NOTE + ------ """ # prog=get_program_from_PID(program) ## try: @@ -1542,6 +2567,30 @@ def get_camera_framerate_by_program(port, program, version=0, testmode=False, ve def get_camera_framerate_by_times(port, starttime, stoptime, version=0, testmode=False): """ + download the framerate for a given camera and each frame in a given time intervall + + INPUT + ------ + port: integer + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + framerate: list + the framerate of the camera for each frame + NOTE + ----- """ OP = IR_tools.get_OP_by_time(time_ns=stoptime) stream=portpathdict[OP]["AEF"+str(port)]+"meta_" @@ -1556,6 +2605,31 @@ def get_camera_framerate_by_times(port, starttime, stoptime, version=0, testmode def get_frametype_by_program(port, program, version=0, testmode=False, verbose=0): """ + download the frametype for a given camera and program for each frame + + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frametype: list + the frametype of the camera for each frame + NOTE + ------ """ ## prog=AKF_1.get_program_from_PID(program) # try: @@ -1573,6 +2647,30 @@ def get_frametype_by_program(port, program, version=0, testmode=False, verbose=0 def get_frametype_by_times(port, starttime, stoptime, version=0, testmode=False): """ + download the frametyp for a given camera and each frame in a given time intervall + + INPUT + ------ + port: integer + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frametype: list + the frametype of the camera for each frame + NOTE + ----- """ OP = IR_tools.get_OP_by_time(time_ns=starttime) stream=portpathdict[OP]["AEF"+str(port)]+"meta_" @@ -1586,6 +2684,31 @@ def get_frametype_by_times(port, starttime, stoptime, version=0, testmode=False) def get_framenumber_by_program(port, program, version=0, testmode=False, verbose=0): """ + download the framenumber for a given camera and program for each frame + + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + framenumber: list + the framenumber of the camera for each frame + NOTE + ------ """ # prog=AKF_1.get_program_from_PID(program) # try: @@ -1603,6 +2726,30 @@ def get_framenumber_by_program(port, program, version=0, testmode=False, verbose def get_framenumber_by_times(port, starttime, stoptime, version=0, testmode=False): """ + download the framenumber for a given camera and each frame in a given time intervall + + INPUT + ------ + port: integer + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + framenumber: list + the framenumber of the camera for each frame + NOTE + ----- """ OP = IR_tools.get_OP_by_time(time_ns=starttime) stream=portpathdict[OP]["AEF"+str(port)]+"meta_" @@ -1616,6 +2763,32 @@ def get_framenumber_by_times(port, starttime, stoptime, version=0, testmode=Fals def get_average_divertor_TC_for_camtemp(port, starttime, stoptime, part="all", TC=0): """ + download of an mean divertor temperature measured by the thermocouples in the divertor with respect to a camera observing this divertor + + INPUT + ------ + port: integer + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + part: string or integer, optional, default "all" + defines the divertor region for the evaluation of the thermocouples (TC)\n + options are 'h' (horizontal), 'v' (vertical), 'hli' (horizontal low iota), 'hhi' (horizontal high iota) 's' or 'single' for only one TC\n + if it is a number, one thermocouple will be used, as counted from low to high iota region(7), going from 0 to 10, 8-10 vor vertical + TC: integer, optional, only for part 's' + TC number if part is 'single' or 's' + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each temperature + temperature: list + averaged TC temperature for each timestamp in time + NOTE + ------ """ urlstart="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.115/" if port==50 or port==51: @@ -1629,8 +2802,10 @@ def get_average_divertor_TC_for_camtemp(port, starttime, stoptime, part="all", T portnr=port+1 offset=(port//10-1)*32+16 else: - print("get_average_divertor_TC_for_camtemp: Error! unknown port") - raise Exception + #print("get_average_divertor_TC_for_camtemp: Error! unknown port") + #raise Exception + logging.warning("get_average_divertor_TC_for_camtemp: Error! unknown port") + return False, [0], [0] if portnr==50: offset=0 elif portnr==51: @@ -1698,6 +2873,20 @@ def get_average_divertor_TC_for_camtemp(port, starttime, stoptime, part="all", T def get_divertor_TC_coordinate(cameraport, TC=0): """ + returns the information of the thermocouple + + INPUT + ------ + cameraport: integer + port number of the AEF port of the camera + TC: integer, default is 0 + TC number going from 0 to 10 + RESULT + ------ + TC_info: dictionary + contains x,y,z information and in which target it is located + NOTE + ------ """ from IR_config_constants import TC_database if cameraport %10==1:##uneven cameraport, even TC port @@ -1710,6 +2899,30 @@ def get_divertor_TC_coordinate(cameraport, TC=0): def get_TC_data_for_port_by_program(port, program, part=0, version=0, verbose=0): """ + download the thermocouple (TC) data for a thermocouple in the port (immersion tube or endoscope) + + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + part: integer, default 0 + 0 or the number of the TC in the port + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + temperature: list + temperature from the TC for each time stamp in time + NOTE + ------ """ ## prog=AKF_1.get_program_from_PID(program) # try: @@ -1727,6 +2940,33 @@ def get_TC_data_for_port_by_program(port, program, part=0, version=0, verbose=0) def get_TC_data_for_port(port, starttime, stoptime, part=0):#, version=0 """ + download the thermocouple (TC) data for a thermocouple in the port (immersion tube or endoscope) + + INPUT + ------ + port: integer + port number of the AEF port of the camera + starttime: integer + time in ns where the search should start + stoptime: integer + time in sn where the search should end + part: integer, default 0 + 0 or the number of the TC in the port + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + temperature: list + temperature from the TC for each time stamp in time + + NOTE + ------ """ urlstart="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.115/DataModuleDesc.19471_DATASTREAM/" #66,67(10),72,73(20),78,79(30),84,85(40/50_2),90,91(50_1) @@ -1785,14 +3025,67 @@ def get_TC_data_for_port(port, starttime, stoptime, part=0):#, version=0 # LUTd={LUT[0][i]: LUT[1][i] for i in range(0, len(LUT[0]))} # return np.vectorize(LUTd.get)(images) -def temp_from_LUT(LUT, images): +def temp_from_LUT(LUT, images): + """ + apply the look up table(LUT) onto images to get temperature images + + INPUT + ------ + LUT: list + list in the form [Temperaure,temperature_error] + images: list + the frames as a list of 2D numpy arrays + RESULT + ------ + images: list + list of numpy arrays + NOTE + ------ + """ return LUT[0, images] def terror_from_LUT(LUT, images): + """ + apply the look up table(LUT) onto images to get temperature error images + + INPUT + ------ + LUT: list + list in the form [Temperaure,temperature_error] + images: list + the frames as a list of 2D numpy arrays + RESULT + ------ + images: list + list of numpy arrays + NOTE + ------ + """ return LUT[1, images] def check_temperature_range(time,maxvalue_C,minvalue_C, port, exposure, cfilter=0): """ + INPUT + ------ + time: integer + nanosecond time stamp to identify the right camera + maxvalue_C: float or integer + the maximum value in the data + minvalue_C: float or integer + the minimum value in the data + port: integer + port number of the AEF port of the camera + exposure: integer + exposure time in microseconds + cfilter: integer, optional, default 0 + the filter for the infratec camera + + RESULT + ------ + no return + NOTE + ------ + not fully functional and slow! """ try: cam=portcamdict[IR_tools.get_OP_by_time(time_ns=time)]['AEF'+str(port)] @@ -1818,6 +3111,45 @@ def check_temperature_range(time,maxvalue_C,minvalue_C, port, exposure, cfilter= def get_temp_from_raw_by_program_V1(portnr, program, time_window=0, emi=0.8,divertorpart="all", version=0, intervalSize=1E9, give_ERROR=False, testmode=False, verbose=0): """ + downloads the raw data of the cameras, the calibration and applies it to the data to deliver the temperature data + This function is outdated + + INPUT + ------ + portnr: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + time_window: list, array, integer or float, optional, default 0 + if integer or float: size of the window in seconds from t1 + if list or array: start and endpoint of the window in seconds from t1 + if 0 the full program will be downloaded (t1 until t6) + emi: float, optional, default 0.8 + the emissivity value for the calibration + divertorpart, optional, default "all" + the region of the divertor, which should be used as the reference temperature + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + intervalSize: integer, optional, default 1e9 (1 second) + size of the intervall to be downloaded in nanoseconds + give_ERROR: boolean, optional default False + switch to turn on the return of error images + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + optional: error_frames: list + a list of frames, 2D numpy arrays + NOTE + ------ """ ## prog=AKF_1.get_program_from_PID(program) @@ -1839,7 +3171,9 @@ def get_temp_from_raw_by_program_V1(portnr, program, time_window=0, emi=0.8,dive tstart=time_window[0] tstop=time_window[1] if tstop<tstart: - raise Exception("get_temp_from_raw_by_program_V1: t_stop before t_start") +# raise Exception("get_temp_from_raw_by_program_V1: t_stop before t_start") + logging.warning("get_temp_from_raw_by_program_V1: t_stop before t_start") + return False, [0], [0] stoptime=int(starttime+tstop*1e9) starttime=int(starttime+tstart*1e9) else: @@ -1874,7 +3208,9 @@ def get_temp_from_raw_by_program_V1(portnr, program, time_window=0, emi=0.8,dive times=times+time del time, images else: - raise Exception("get_temp_from_raw_by_program_V1: cannot download the raw images") +# raise Exception("get_temp_from_raw_by_program_V1: cannot download the raw images") + logging.warning("get_temp_from_raw_by_program_V1: cannot download the raw images") + return False, [0], [0] else: raw_dl=download_raw_images_by_times(portnr, starttime, stoptime, version, intervalSize, testmode=testmode, verbose=verbose-1) if raw_dl[0]: @@ -1907,6 +3243,43 @@ def get_temp_from_raw_by_program_V2(portnr, program, time_window=0, emi=0.8, ver threads=1, give_ERROR=False,use_firstframe_as_background=False, verbose=0): """ + downloads the raw data of the cameras, the calibration and applies it to the data to deliver the temperature data + + INPUT + ------ + portnr: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + time_window: list, array, integer or float, optional, default 0 + if integer or float: size of the window in seconds from t1 + if list or array: start and endpoint of the window in seconds from t1 + if 0 the full program will be downloaded (t1 until t6) + emi: float, optional, default 0.8 + the emissivity value for the calibration + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + threads: integer, optional, default 1 + number of parallel threads to download the images + (approx. max. 2x number of cores and max. timeintervall/0.5 for best performance) + give_ERROR: boolean, optional default False + switch to turn on the return of error images + use_firstframe_as_background: boolean + in case the background frames are not available, the first frame can be used + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + optional: error_frames: list + a list of frames, 2D numpy arrays + NOTE + ------ """ return get_temp_from_raw_by_program(portnr, program, time_window=time_window, emi=emi, T_version=2, version=version, @@ -1919,11 +3292,59 @@ def get_temp_from_raw_by_program(portnr, program, time_window=0, emi=0.8, T_vers use_firstframe_as_background=False, back_emissivity=0.8, verbose=0, testmode=False, framerate='max'): """ + downloads the raw data of the cameras, the calibration and applies it to the data to deliver the temperature data + several threads can be used to download in parallel the data + + INPUT + ------ + portnr: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + time_window: list, array, integer or float, optional, default 0 + if integer or float: size of the window in seconds from t1 + if list or array: start and endpoint of the window in seconds from t1 + if 0 the full program will be downloaded (t1 until t6) + emi: float, optional, default 0.8 + the emissivity value for the calibration + T_version: integer, default 2 + set the version of the calibration method, see more on the QRT wiki + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + threads: integer, optional, default 1 + number of parallel threads to download the images + (approx. max. 2x number of cores and max. timeintervall/0.5 for best performance) + give_ERROR: boolean, optional default False + switch to turn on the return of error images + use_firstframe_as_background: boolean + in case the background frames are not available, the first frame can be used + back_emissivity: float, optional, default 0.8 + set the emissivity value for the background region for the calibration + verbose: integer, optional, default 0 + feedback level (details of print messages) + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + framerate: string or integer, optional, default 'max' + sets the framerate to downsample the data. normal rate is 100. + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays, values are in Kelvin + optional: error_frames: list + a list of frames, 2D numpy arrays + NOTE + ------ """ FLIR=False trigger_exist, t0, t1, t6 = get_trigger_from_PID(program, portnr, testmode, verbose=verbose-1) if not trigger_exist: - raise Exception("get_temp_from_raw_by_program: unknown Program") +# raise Exception("get_temp_from_raw_by_program: unknown Program") + logging.warning("get_temp_from_raw_by_program: unknown Program") + return False, [0], [0] success=True OP = IR_tools.get_OP_by_time(time_ns=t1) @@ -1945,8 +3366,9 @@ def get_temp_from_raw_by_program(portnr, program, time_window=0, emi=0.8, T_vers FLIR=False portnr=int(portnr) else: - raise Exception("get_temp_from_raw_by_program: the given port is neither a number or a valid String!") - +# raise Exception("get_temp_from_raw_by_program: the given port is neither a number or a valid String!") + logging.warning("get_temp_from_raw_by_program: the given port is neither a number or a valid String!") + return False, [0], [0] if FLIR: # --- Glens FLIR camera --- # @@ -2052,7 +3474,8 @@ def get_temp_from_raw_by_program(portnr, program, time_window=0, emi=0.8, T_vers if exist: frames = [im.swapaxes(0,1) for im in frames] if not exist: - raise Warning("get_temp_from_raw_by_program: data not found in database!") +# raise Warning("get_temp_from_raw_by_program: data not found in database!") + logging.warning("get_temp_from_raw_by_program: data not found in database!") return False, [0], [0], False if verbose > 0: print(datetime.datetime.now(), "get_temp_from_raw_by_program: raw download finished") @@ -2110,6 +3533,47 @@ def get_temp_from_raw_by_program_fullthreads(portnr, program, time_window=0, emi give_ERROR=False, check_range=True, testmode=False, verbose=0): """ + downloads the raw data and converting it to temeprature images + + INPUT + ------ + portnr: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + time_window: list, array, integer or float, optional, default 0 + if integer or float: size of the window in seconds from t1 + if list or array: start and endpoint of the window in seconds from t1 + if 0 the full program will be downloaded (t1 until t6) + emi: float, optional, default 0.8 + the emissivity value for the calibration + T_version: integer, default 2 + set the version of the calibration method, see more on the QRT wiki + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + threads: integer, optional, default 1 + number of parallel threads to download the images + (approx. max. 2x number of cores and max. timeintervall/0.5 for best performance) + give_ERROR: boolean, optional default False + switch to turn on the return of error images + check_range: boolean, optional, default True + switch to turn the temperature range check on or off + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays, values are in Kelvin + optional: error_frames: list + a list of frames, 2D numpy arrays + NOTE + ------ """ if threads == 1 or not fastDL: return get_temp_from_raw_by_program(portnr, program, time_window, emi, @@ -2127,7 +3591,10 @@ def get_temp_from_raw_by_program_fullthreads(portnr, program, time_window=0, emi portnr=int(portnr.split("AEF")[1]) goon=True except Exception as E: - raise Exception("get_temp_from_raw_by_program_fullthreads: unknown Port!"+E) +# raise Exception("get_temp_from_raw_by_program_fullthreads: unknown Port!"+E) + logging.warning("get_temp_from_raw_by_program_fullthreads: unknown Port!"+E) + return False, [0], [0] + elif type(portnr) is int: FLIR = False goon = True @@ -2155,7 +3622,9 @@ def get_temp_from_raw_by_program_fullthreads(portnr, program, time_window=0, emi tstart = time_window[0] tstop = time_window[1] if tstop < tstart: - raise Exception("get_temp_from_raw_by_program_fullthreads: t_stop before t_start") +# raise Exception("get_temp_from_raw_by_program_fullthreads: t_stop before t_start") + logging.warning("get_temp_from_raw_by_program_fullthreads: t_stop before t_start") + return False, [0], [0] stdate = datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart) stdate = stdate.isoformat() enddate = datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop) @@ -2236,6 +3705,47 @@ def get_nuced_raw_by_program(portnr, program, time_window=0,version=0, emi=1, T_version=2, threads=1, give_ERROR=False, testmode=False, verbose=0): """ + download the raw data and applies the non uniformity correction to make the image understandable + + INPUT + ------ + portnr: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + time_window: list, array, integer or float, optional, default 0 + if integer or float: size of the window in seconds from t1 + if list or array: start and endpoint of the window in seconds from t1 + if 0 the full program will be downloaded (t1 until t6) + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + emi: float, optional, default 0.8 + the emissivity value for the calibration + T_version: integer, default 2 + set the version of the calibration method, see more on the QRT wiki + threads: integer, default 1 + number of parallel threads to download the images + (approx. max. 2x number of cores and max. timeintervall/0.5 for best performance) + give_ERROR: boolean, optional default False + switch to turn on the return of error images + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + valid: boolean + status wheather the data can be used + optional frames_error: list + a list of frames, 2D numpy arrays + NOTE + ------ """ # get calibration data exist, background, LUT,refT, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error=get_calib_data(portnr, program=program, emissivity=emi, T_version=T_version, version=version, testmode=testmode, verbose=verbose-1) @@ -2249,7 +3759,9 @@ def get_nuced_raw_by_program(portnr, program, time_window=0,version=0, if exist: frames=[im.swapaxes(0,1) for im in frames] else: - raise Exception("get_nuced_raw_by_program: no data found") +# raise Exception("get_nuced_raw_by_program: no data found") + logging.warning("get_nuced_raw_by_program: no data found") + return False, [0], [0], False if verbose > 0: print(datetime.datetime.now(), "get_nuced_raw_by_program: raw download finished") FOV=get_FOV_mask(portnr) @@ -2277,6 +3789,47 @@ def get_nuced_raw_by_program_fullthreads(portnr, program, time_window=0, emi=0.8 give_ERROR=False, check_range=True, testmode=False, verbose=0): """ + download the raw data and applies the non uniformity correction to make the image understandable by using threads for all steps + + INPUT + ------ + portnr: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + time_window: list, array, integer or float, optional, default 0 + if integer or float: size of the window in seconds from t1 + if list or array: start and endpoint of the window in seconds from t1 + if 0 the full program will be downloaded (t1 until t6) + emi: float, optional, default 0.8 + the emissivity value for the calibration + T_version: integer, default 2 + set the version of the calibration method, see more on the QRT wiki + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + threads: integer, default 1 + number of parallel threads to download the images + (approx. max. 2x number of cores and max. timeintervall/0.5 for best performance) + give_ERROR: boolean, optional default False + switch to turn on the return of error images + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays + valid: boolean + status wheather the data can be used + optional frames_error: list + a list of frames, 2D numpy arrays + NOTE + ------ """ # prog=get_program_from_PID(program) # if prog[0]: @@ -2310,7 +3863,9 @@ def get_nuced_raw_by_program_fullthreads(portnr, program, time_window=0, emi=0.8 tstart = time_window[0] tstop = time_window[1] if tstop < tstart: - raise Exception("get_nuced_raw_by_program_fullthreads: t_stop before t_start") +# raise Exception("get_nuced_raw_by_program_fullthreads: t_stop before t_start") + logging.warning("get_nuced_raw_by_program_fullthreads: t_stop before t_start") + return False, [0], [0], False stdate = datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart) stdate = stdate.isoformat() enddate = datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop) @@ -2378,8 +3933,43 @@ def get_nuced_raw_by_program_fullthreads(portnr, program, time_window=0, emi=0.8 def apply_calib_on_raw(images, background, LUT,refT=28.5, gain=0, offset=0, gain_error=0, offset_error=0, fullbackground=False, give_ERROR=False, verbose=0): """ - apply_calib_on_raw - return success, images and if give_Error: errorimages + + applies the calibration onto the given raw images and return the temperature images + + INPUT + ------ + images: list + a list of frames, 2D numpy arrays + background: integer or 2D array + the value of the background for calibration, which get subtracted from the images + LUT: list + list or array containing the look up table in the form of [[DL],[temperature],[error]] + refT: float, default 28.5 + reference temperature in degree celcius of the background + gain: numpy array + the gain image from the calibration to convert the counts into digital level + offset: numpy array + the offset image from the calibration to convert the counts into digital level + gain_error: numpy array + the gain error image from the calibration to convert the counts into digital level + offset_error: numpy array + the offset error image from the calibration to convert the counts into digital level + fullbackground: boolean, default False + switch to set the treatment of the backgroun input + give_ERROR: boolean, optional default False + switch to turn on the return of error images + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + frames: list + a list of frames, 2D numpy arrays, values are in Kelvin + optional frames_error: list + a list of frames, 2D numpy arrays + NOTE + ------ """ try: #images=np.array(raw,dtype=np.uint16) @@ -2430,13 +4020,39 @@ def apply_calib_on_raw(images, background, LUT,refT=28.5, gain=0, offset=0, gain images[i]=(images[i]+(refT+273.15)).astype(np.float32) return True, images except Exception as E: - raise Warning('apply_calib_on_raw: '+E) -# return False, [0] +# raise Warning('apply_calib_on_raw: '+E) + logging.warning('apply_calib_on_raw: '+E) + return False, [0] def apply_NUC(images, gain, offset, gain_error=0, offset_error=0, give_error=False): """ apply_NUC(images=numpy array(time,width,height), gain, offset =numpy array(width,height) ) + + applies the non-uniformity correction onto the given raw images + + INPUT + ------ + frames: list + a list of frames, 2D numpy arrays + gain: numpy array + the gain image from the calibration to convert the counts into digital level + offset: numpy array + the offset image from the calibration to convert the counts into digital level + gain_error: numpy array + the gain error image from the calibration to convert the counts into digital level + offset_error: numpy array + the offset error image from the calibration to convert the counts into digital level + give_error + + RESULT + ------ + frames: list + a list of frames, 2D numpy arrays + optional frames_error: list + a list of frames, 2D numpy arrays + NOTE + ------ """ try: if give_error: @@ -2457,6 +4073,22 @@ def apply_NUC(images, gain, offset, gain_error=0, offset_error=0, give_error=Fal def get_cooling_water_temperature_by_time(time_ns): """ + downloads the information of the water temperture in the cooling cycle of W7-X + + INPUT + ------ + time_ns: integer + nanosecond time stamp to identify the time window + RESULT + ------ + exist: boolean + True if data was found, False if not + inlet: float + the water temperature of the inflow water + outlet: float + the water temperature of the outgoing water + NOTE + ------ """ starttime=int(time_ns-3e9) stoptime=int(time_ns+3e9) @@ -2490,8 +4122,59 @@ def get_cooling_water_temperature_by_time(time_ns): def get_calib_data(port, program=None, t0=None, t1=None, t6=None, emissivity=0.8, T_version=2, version=0, back_emissivity=0.82, testmode=False, verbose=0): """ - get_calib_data - return exist, background, LUT,refT, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error + get_calib_data downloads the calibration data for the application onto the images + + + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string, default None + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + t0: integer, default None + nanoseconds timestamp of the t0 trigger, start of the program + t1: integer, default None + nanoseconds timestamp of the t1 trigger, start of the heating + t6: integer, default None + nanoseconds timestamp of the t6 trigger, end of the program + emissivity: float, optional, default 0.8 + the emissivity value for the calibration + T_version: intger, default 2, + set the calibration version, in version 1 the full background image is subtracted\n in version 2 the background value is subtracted, version 3 + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + back_emissivity: float, optional, default 0.82, experimental + emissivity value for the background treatment, only needed for T_version 3 + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + background: integer or numpy array + background image for calibration method 1 and a value for method 2 + LUT: list + the look up table for the temperature calibration, form [[Digital level],[temperature],[error]] + refT: float + reference temperature of the background for the calibration + gain: numpy array + the gain image from the calibration to convert the counts into digital level + offset: numpy array + the offset image from the calibration to convert the counts into digital level + badpixels: list + list of badpixels + t_exp: integer + exposure time in microseconds + cfilter: integer or string, only needed for port 50 + filter number of the Infratec camera in port AEF50 + gain_error: numpy array + the gain error image from the calibration to convert the counts into digital level for the error + offset_error: numpy array + the offset error image from the calibration to convert the counts into digital level for the error + NOTE + ------ """ if t0 is None or t1 is None or t6 is None: if program is None: @@ -2500,12 +4183,16 @@ def get_calib_data(port, program=None, t0=None, t1=None, t6=None, emissivity=0.8 elif t1 is not None: program = AKF_2.get_program_id(t1) else: - raise Exception('get_calib_data: ERROR! Missing time definition!') +# raise Exception('get_calib_data: ERROR! Missing time definition!') # print(' function requires either a program ID or the trigger times t0, t1 and t6') + logging.warning('get_calib_data: ERROR! Missing time definition!') + return False, 0, [], 0, [], [], [], 0, 0, [], [] # get start and end times from program ID exist, t0, t1, t6 = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if not exist: - raise Exception('get_calib_data: ERROR! program {0} not found!'.format(program)) +# raise Exception('get_calib_data: ERROR! program {0} not found!'.format(program)) + logging.warning(f'get_calib_data: ERROR! program {program} not found!') + return False, 0, [], 0, [], [], [], 0, 0, [], [] # determin camera in this port and OP OP = IR_tools.get_OP_by_time(time_ns=t0) @@ -2520,8 +4207,9 @@ def get_calib_data(port, program=None, t0=None, t1=None, t6=None, emissivity=0.8 cfilter = cf[2] else: cfilter = -1 - raise Warning("get_calib_data: Filter not found for INFRATEC Camera for Program "+AKF_2.get_program_id(t0)) -# return False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +# raise Warning("get_calib_data: Filter not found for INFRATEC Camera for Program "+AKF_2.get_program_id(t0)) + logging.warning(f"get_calib_data: Filter not found for INFRATEC Camera for Program {AKF_2.get_program_id(t0)}") + return False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 else: cfilter=0 @@ -2531,8 +4219,9 @@ def get_calib_data(port, program=None, t0=None, t1=None, t6=None, emissivity=0.8 t_exp = int(expo_DL[2][0]) del expo_DL else: - raise Warning("get_calib_data: no exposure time found") -# return False, 0, 0, 0, 0, 0, 0, 0, cfilter, 0, 0 +# raise Warning("get_calib_data: no exposure time found") + logging.warning("get_calib_data: no exposure time found") + return False, 0, 0, 0, 0, 0, 0, 0, cfilter, 0, 0 # get Look-Up-Table (LUT) time = int(TimeToNs([2017,9,26], [8, 0, 0, 0])) @@ -2541,8 +4230,9 @@ def get_calib_data(port, program=None, t0=None, t1=None, t6=None, emissivity=0.8 LUT=LUT_DL[1] del LUT_DL else: - raise Warning("get_calib_data: no LUT found") -# return False, 0, 0, 0, 0, 0, 0, t_exp, cfilter, 0, 0 +# raise Warning("get_calib_data: no LUT found") + logging.warning("get_calib_data: no LUT found") + return False, 0, 0, 0, 0, 0, 0, t_exp, cfilter, 0, 0 # treat cameras with and w/o filters differently if cfilter==0 and not Camera.split("_")[0] == "Infratec": @@ -2551,8 +4241,9 @@ def get_calib_data(port, program=None, t0=None, t1=None, t6=None, emissivity=0.8 print('get_calib_data: getting NUC arrays') NUC_DL=get_NUC_by_times(port, t0, t1, t_exp, version, testmode, verbose=verbose-1)#download_NUC_by_times(port, t0, t1, t_exp, version) if NUC_DL[0]==False: - raise Warning("get_calib_data: NUC was not found") -# return False, 0, LUT, 0, 0, 0, 0, t_exp, cfilter, 0, 0 +# raise Warning("get_calib_data: NUC was not found") + logging.warning("get_calib_data: NUC was not found") + return False, 0, LUT, 0, 0, 0, 0, t_exp, cfilter, 0, 0 # extract NUC elements gain = [] offset = [] @@ -2590,8 +4281,9 @@ def get_calib_data(port, program=None, t0=None, t1=None, t6=None, emissivity=0.8 bim = apply_NUC([frames[0]], gain, offset) background = get_average_background_recangle(port, bim[0]) else: - raise Warning("get_calib_data: cannot find the first frame of the discharge, reconstruction failed") -# return False, 0, LUT, 0, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error +# raise Warning("get_calib_data: cannot find the first frame of the discharge, reconstruction failed") + logging.warning("get_calib_data: cannot find the first frame of the discharge, reconstruction failed") + return False, 0, LUT, 0, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error else: # --- Infratec Camera case --- gain = 0 @@ -2606,9 +4298,13 @@ def get_calib_data(port, program=None, t0=None, t1=None, t6=None, emissivity=0.8 elif OP == "OP1.2b": back_DL = get_NUCed_background_by_times(port, t0, t1, t_exp, cfilter, gain, offset, version, testmode, verbose=verbose-1) else: - raise Exception("get_calib_data: unknown Operation Phase or calibration method not implemented for this campaign") +# raise Exception("get_calib_data: unknown Operation Phase or calibration method not implemented for this campaign") + logging.warning("get_calib_data: unknown Operation Phase or calibration method not implemented for this campaign") + return False, 0, [], 0, [], [], [], 0, 0, [], [] if not back_DL[0]: - raise Warning("get_calib_data: no background image found") +# raise Warning("get_calib_data: no background image found") + logging.warning("get_calib_data: no background image found") + return False, 0, [], 0, [], [], [], 0, 0, [], [] # return False, 0, LUT, 0, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error background = back_DL[2] @@ -2627,7 +4323,9 @@ def get_calib_data(port, program=None, t0=None, t1=None, t6=None, emissivity=0.8 if TCT_dl[0]: refT = np.average(TCT_dl[2]) else: - raise Exception("get_calib_data: Unable to find thermocouple data") +# raise Exception("get_calib_data: Unable to find thermocouple data") + logging.warning("get_calib_data: Unable to find thermocouple data") + return elif T_version == 2: if verbose > 0: print('get_calib_data: use temperature calibration version 2') @@ -2663,7 +4361,9 @@ def get_calib_data(port, program=None, t0=None, t1=None, t6=None, emissivity=0.8 # back_off=estimate_offset(port, program) # background=(background-back_off)/(back_emissivity)+back_off else: - raise Warning("get_calib_data: Unknown temperature calibration method") +# raise Warning("get_calib_data: Unknown temperature calibration method") + logging.warning("get_calib_data: Unknown temperature calibration method") + return False, 0, [], 0, [], [], [], 0, 0, [], [] # return False, background, LUT, 0, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error # get bad pixels @@ -2687,6 +4387,32 @@ def get_calib_data(port, program=None, t0=None, t1=None, t6=None, emissivity=0.8 def find_badpixels(port, gain, offset, init_bp_list=None, niterations=3, tolerance=10, plot_it=False, verbose=0): """ + find the bad pixels by searching outliers in the gain and offset image + + INPUT + ------ + port: integer + port number of the AEF port of the camera + gain: numpy array + the gain image from the calibration to convert the counts into digital level + offset: numpy array + the offset image from the calibration to convert the counts into digital level + init_bp_list: list, default None + initial bad pixel list + niterations: intger, default 3 + number of interation in the process of finding bad pixels + tolerance: integer, default 10 + tolerance value for the idenfication of outlier in the images to identify the bad pixels + plot_it: boolean, default False, + if True the result of the bad pixel finding will be plotted + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + badpixels: numpy array + list of the bad pixels + NOTE + ------ """ badpixels = np.zeros(np.shape(gain)) # certain_bads=np.zeros(np.shape(gain)) @@ -2729,6 +4455,17 @@ def find_badpixels(port, gain, offset, init_bp_list=None, niterations=3, def make_FOV_mask(port): """ + creates the field of view image for the requested port and saves it locally + + INPUT + ------ + port: integer + port number of the AEF port of the camera + RESULT + ------ + no return + NOTE + ------ """ points=valid_FOV_circle[port] @@ -2773,6 +4510,18 @@ def make_FOV_mask(port): def get_FOV_mask(port): """ + loads the locally saved field of view images for the requested port, see also make_FOV_mask + + INPUT + ------ + port: integer + port number of the AEF port of the camera + RESULT + ------ + FOV: numpy array + the image showing the field of view with 1 and 0 for the region of the image which is outside the field of view + NOTE + ------ """ try: if config_path=="": @@ -2790,6 +4539,20 @@ def get_FOV_mask(port): def get_background(port, image): """ + extracts from a given background image the minimum in the field of view + + INPUT + ------ + port: integer + port number of the AEF port of the camera + image: numpy array + 2D numpy arrays + RESULT + ------ + background: integer (or float if image contains float values) + returns the background value, extracted from the given image + NOTE + ------ """ try: # points=valid_background_rectangle[port] @@ -2802,6 +4565,20 @@ def get_background(port, image): def get_average_background_recangle(port, image): """ + extracts from a given background image the minimum in a defined rectangular region + + INPUT + ------ + port: integer + port number of the AEF port of the camera + image: numpy array + 2D numpy arrays + RESULT + ------ + background: integer (or float if image contains float values) + returns the background value, extracted from the given image + NOTE + ------ """ try: points = valid_background_rectangle[port] @@ -2814,6 +4591,24 @@ def get_average_background_recangle(port, image): def estimate_offset(port, program, plot_it=False, verbose=0): """ + estimates the offset value based on the background values for different exposure times + + INPUT + ------ + port: integer + port number of the AEF port of the camera + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + plot_it: boolean, default False + if True the result of the bad pixel finding will be plotted + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + Offset: integer + estimated offset value + NOTE + ------ """ explist = [1, 2, 3, 4, 5, 6, 7, 8, 9] meanlist = [] @@ -2841,16 +4636,46 @@ def estimate_offset(port, program, plot_it=False, verbose=0): return -1 def download_heatflux_by_program(port, program, time_window=0, testmode=False, - version=0, verbose=0, givealpha=False,request=True,moffset=0):#threads=1, + version=0, verbose=0, givealpha=False,request=True):#threads=1, """ - port: string or integer - program: string - time_window: integer, float or list/numpy array of interger/float, single value: t1 until so many seconds, two values, t1+ first value until t1+ second value - threads: integer, number of threads for parallel download - testmode: True to load data from testarchive - version: integer, version number for 0 the highest version will be used - verbose: interger, level of outputs - returns downlad_heatflux_by_times + download the heatflux for a given program and port, if the data is not available a request is made for the automated evaluation of the heat flux\n + if the data is not available or already requested a message will be returned in the console\n + + For extended use of the heat flux data see also the following functions:\n + download_heatflux_mapping_reference --> for a better interpretation of the data with respect to the used coordinate system + download_heatflux_scene_model_reference --> for remapping of the data into W7-X coordinates and pixel coordiantes + extract_heatflux_profile_from_DL --> to extract a profile out of the data returned by this funciton + + + INPUT + ------ + port: string or integer + the camera port, for which the data is requested + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + time_window: integer, float or list/numpy array of interger/float, + time window for the heat flux, single value: t1 until so many seconds, two values, t1+ first value until t1+ second value + testmode: boolean, default False + True to load data from testarchive + version: integer, default 0 + version number for 0 the highest version will be used + verbose: integer, optional, default 0 + feedback level (details of print messages) + givealpha: boolean, default False + switch to turn on or off the return of the alpha image from the THEODOR calculation, it wil be returned as first image + request: boolean, default True + switch to turn on or off the request of data if the data is not available in the database + RESULT + ------ + (returns downlad_heatflux_by_times) + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame + frames: list + a list of frames, 2D numpy arrays, first frame is the alpha frame if it is requested + NOTE + ------ """ # prog=get_program_from_PID(program) # if prog[0]: @@ -2873,31 +4698,61 @@ def download_heatflux_by_program(port, program, time_window=0, testmode=False, tstart=int(t1) tend=int(t1+time_window[0]*1e9) except Exception as E: - raise Exception("download_heatflux_by_program: an Exception occured interpretating the time_window: ", E) +# raise Exception("download_heatflux_by_program: an Exception occured interpretating the time_window: ", E) + logging.warning("download_heatflux_by_program: an Exception occured interpretating the time_window: "+E) + return False, [0], [0] elif type(time_window)==int or type(time_window)==float: tstart=int(t1) tend=int(t1+time_window*1e9) else: - raise Exception("download_heatflux_by_program: given time_window type is not supported") +# raise Exception("download_heatflux_by_program: given time_window type is not supported") + logging.warning("download_heatflux_by_program: given time_window type is not supported") + return False, [0], [0] if givealpha: tstart=tstart-1.1e9 return download_heatflux_by_times(port=port, tstart=tstart, tend=tend, time_window=0, testmode=testmode, version=version, verbose=verbose, - request=request,moffset=moffset) + request=request) -def download_heatflux_by_times(port, tstart, tend, time_window=0, threads=1, testmode=False, - version=0, verbose=0,request=True,moffset=0): - """ - port: string or integer - tstart: int64 timestamp in ns from which point the download should start - tend: int64 timestamp in ns - threads: integer, number of threads for parallel download - testmode: True to load data from testarchive - version: integer, version number for 0 the highest version will be used - verbose: interger, level of outputs - returns found, time, images +def download_heatflux_by_times(port, tstart, tend, time_window=0, testmode=False, + version=0, verbose=0,request=True,request_ALL=False):#threads=1, ,moffset=0 + """ + + download the heatflux for a given start, endtime and port, if the data is not available a request is made for the automated evaluation of the heat flux\n + if the data is not available or already requested a message will be returned in the console + + INPUT + ------ + port: string or integer + the camera port, for which the data is requested + tstart: integer + time in ns where the search should start + tend: integer + time in sn where the search should end + time_window: integer, float or list/numpy array of interger/float, + time window for the heat flux, single value: t1 until so many seconds, two values, t1+ first value until t1+ second value + testmode: boolean, default False + True to load data from testarchive + version: integer, default 0 + version number for 0 the highest version will be used + verbose: integer, optional, default 0 + feedback level (details of print messages) + request: boolean, default True + switch to turn on or off the request of data if the data is not available in the database + request_ALL: boolean, optional, default False + switch to turn on the request of data for all AEF ports + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + a list containing the time stamp of each frame, time is in nanoseconds UTC + frames: list + a list of frames, 2D numpy arrays, values in the images are in W/m2 + NOTE + ------ """ if testmode: base=testarchivepath @@ -2914,7 +4769,9 @@ def download_heatflux_by_times(port, tstart, tend, time_window=0, threads=1, tes port=int(port.split("AEF")[1]) goon=True except Exception as E: - raise Exception("download_heatflux_by_times: unknown Port!"+E) +# raise Exception("download_heatflux_by_times: unknown Port!"+E) + logging.warning("download_heatflux_by_times: unknown Port!"+E) + return False, [0], [0] elif isinstance(port, (int, np.integer, float, np.float)): FLIR=False goon=True @@ -2940,6 +4797,7 @@ def download_heatflux_by_times(port, tstart, tend, time_window=0, threads=1, tes if verbose > 0: print(datetime.datetime.now(), "download_heatflux_by_times: heat flux download started") if False: ## does not work, the png pictures are interpretated as colors, values are lost + threads = 1 exist, time, frames=download_images_by_time_via_png(larchivepath, starttime=tstart, stoptime=tend, time_window=time_window, version=version, threads=threads, verbose=verbose-1) else: exist, time, frames=download_images_by_times(larchivepath, starttime=tstart, stoptime=tend, verbose=verbose-1) @@ -2962,9 +4820,13 @@ def download_heatflux_by_times(port, tstart, tend, time_window=0, threads=1, tes cam_progs_ig,reasons=IR_tools.get_work_list(heatflux_requestlist_path, typ='ignore') except: cam_progs_ig=[] - if (programid, "AEF"+str(port)) not in cam_progs and (programid, "AEF"+str(port)) not in cam_progs_ig: - f=open(heatflux_requestlist_path+str(now.year)+str(now.month+moffset)+"_q_requests.txt", 'a') - f.write(programid+"\tAEF"+str(port)+"\n") + cam_p = cam_progs + cam_progs_ig + if (programid, "AEF"+str(port)) not in cam_p and (programid, "ALL") not in cam_p: + f=open(heatflux_requestlist_path+"Auto_q_requests.txt", 'a') + if request_ALL: + f.write(programid+"\tALL\n") + else: + f.write(programid+"\tAEF"+str(port)+"\n") f.close() if verbose > 0: print("download_heatflux_by_times: heat flux calculation request logged for automatic processing (within ca. 1 day)") @@ -2979,7 +4841,28 @@ def download_heatflux_by_times(port, tstart, tend, time_window=0, threads=1, tes def download_heatflux_mapping_reference(timepoint=None, version=0, testmode=False, get_thickness=False, verbose=0): """ - return exist(boolean) and dictonary of the mappings informations + returns the mapping informations for the heatflux to relate each pixel in the heat flux image to different additional parameters. e.g., X and Y coordiante of the divertor mapping + + INPUT + ------ + timepoint: integer, optional, default None + set the timepoint in nanoseconds where/when to search for the mapping in the database + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + get_thickness: boolean, optional, default False + turns on the return of the thickness + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + mappings: dictionary + the mapping informations containing: surface coordinate s, divertor coordinate system X,Y, Finger coordiante system X, Y, Finger Id, Thickness, target, Target_Module\n all coordinates in meter + NOTE + ------ """ if timepoint == None: timepoint = int(TimeToNs([2017,9,26], [8, 0, 0, 0])) @@ -3111,7 +4994,30 @@ def download_heatflux_mapping_reference(timepoint=None, version=0, testmode=Fals def download_heatflux_scene_model_reference(port, timepoint=None, program=None, version=0, testmode=False, verbose=0): """ - return exist(boolean) and dictonary of the mappings informations + returns the mapping informations based on the scene model for the heatflux to relate each pixel in the heat flux image to different additional parameters. e.g., X and Y coordiante in the original image or the CAD geometry in X,Y,Z + + INPUT + ------ + port: integer or string + port of the AEF port of the camera + timepoint: integer, optional, default None + set the timepoint in nanoseconds where/when to search for the mapping in the database + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + testmode: boolean, optional, default False + if True, the data will be loaded from the test archive + get_thickness: boolean, optional, default False + turns on the return of the thickness + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + mappings: dictionary + the mapping informations containing: most likely Pixel coordiante X,Y, CAD X,Y,Z and the component id as PFC, \n all coordinates in meter + NOTE + ------ """ if type(port) == int or type(port) == float: portnam = "AEF"+str(port) @@ -3174,8 +5080,24 @@ def download_heatflux_scene_model_reference(port, timepoint=None, program=None, def give_finger_ID(profile, finger=None, verbose=0): """ - profile: string, integer or float, string: "TM3h_5_5" or "1605" or "16.05" or "5", similar for float and integer. single number only if the finger is given! - finger: string or integer, string "TM3h_5" or "16" or 16 or 1600 or None + returns for a given profile (and finger) information the finger ID which is internally used in the mappings + + INPUT + ------ + profile: string, integer or float, + string: "TM3h_5_5" or "1605" or "16.05" or "5", similar for float and integer. single number only if the finger is given! + finger: string or integer, optional, default None + string "TM3h_5" or "16" or 16 or 1600 or None + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + success: boolean + gives the information whether the idenfication of the fingerID went well or not + fingerID: integer + the finger idendifikation in the form of XXXYY with XXX the fingernumber *100 and YY the profile number, starting with 0 + NOTE + ------ """ ### offsets={ @@ -3262,12 +5184,48 @@ def give_finger_ID(profile, finger=None, verbose=0): def get_heatflux_profile(port, profile, finger=None, timepoint=None, program=None, tstart=None, tend=None, testmode=False, version=0, verbose=0): """ - returns exist, time, s in m, heat flux in W/m2 + returns a heat flux profile for a given port, profile (and finger) and timepoint in a program/time intervall defined by tstart and tend + \n it is a combination of the following functions:\n + download_heatflux_by_prrogram or download_heatflux_by_times\n + extract_heatflux_profile_from_DL, since it is called in the end\n + and also give_finger_ID for the interpretation of the profile/finger input\n - parameters: + in case you want several profiles, please use the function extract_heatflux_profile_from_DL to lower the amount of downloaded data (speed things up) + + INPUT + ------ port: string or integer - profile: string, integer or float, string: "TM3h_5_5" or "1605" or "16.05" or "5", similar for float and integer. single number only if the finger is given! - finger: string or integer, string "TM3h_5" or "16" + the port for which the data is requested, normally a AEF port + profile: string, integer or float, + string: "TM3h_5_5" or "1605" or "16.05" or "5", similar for float and integer. single number only if the finger is given! + finger: string or integer, + string "TM3h_5" or "16" + timepoint: integer, default None + the timepoint for the profile in seconds from T1 trigger + program: string + program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016' + tstart: integer + time in ns where the search should start + tend: integer + time in ns where the search should end + testmode: boolean, default False + True to load data from testarchive + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + verbose: integer, optional, default 0 + feedback level (details of print messages) + RESULT + ------ + exist: boolean + True if data was found, False if not + time: list + timestamp of the profile in seconds + s: list of float + surface coordinate in meter for each profile position + heatflux: list of float + the heat flux in in W/m2 for each profile position + NOTE + ------ """ ### checking whether input is valid and whether the download can be started if program == None and tstart == None: @@ -3322,15 +5280,18 @@ def get_heatflux_profile(port, profile, finger=None, timepoint=None, program=Non port = int(port.split("AEF")[1]) goon = True except Exception as E: - raise Warning("get_heatflux_profile: unknown Port!"+E) -# return False, 0, 0, 0 +# raise Warning("get_heatflux_profile: unknown Port!"+E) + logging.warning("get_heatflux_profile: unknown Port!"+E) + return False, 0, 0, 0 elif isinstance(port, (int, np.integer, float, np.float)): FLIR = False goon = True else: goon = False if not goon or FLIR: - raise Exception("get_heatflux_profile: Unknown port, FLIR data is unsupported at the moment") +# raise Exception("get_heatflux_profile: Unknown port, FLIR data is unsupported at the moment") + logging.warning("get_heatflux_profile: Unknown port, FLIR data is unsupported at the moment") + return False, 0, 0, 0 else: ### if the program goes up to here, time okay, finger okay, port okay, lets download it exist, timo, frames=download_heatflux_by_times(port, tstart, tend, time_window=0, threads=1, testmode=testmode, version=version, verbose=verbose-1) @@ -3344,8 +5305,41 @@ def get_heatflux_profile(port, profile, finger=None, timepoint=None, program=Non def extract_heatflux_profile_from_DL(time, images, profile, finger=None, time_window=None, inputchecked=False, verbose=0, testmode=False): """ - return exist, time, s in m, heat flux in W/m2 - time_point default (None) returns entire shot. + extracts from given heat flux images the requested profile for the given time window + + INPUT + ------ + time: list + the timevector of the data in seconds + images: list + list of frames in form of numpy arrays with the heat flux data + profile: string, integer or float, + string: "TM3h_5_5" or "1605" or "16.05" or "5", similar for float and integer. single number only if the finger is given! + finger: string or integer, + string "TM3h_5" or "16" + time_window: integer, float or list/numpy array + defines the timewindow, if only a number is give or an array/list with only one number, it searches one profile for the given timevector, \n + if a list or array is given, several profiles will be returned, in case of None, the full program will be returned + inputchecked: boolean, optional, default False + defines whether the input parameters have been checked for correct format or not (profile and finger) + verbose: integer, optional, default 0 + feedback level (details of print messages) + testmode: boolean, optional, default 0 + if True, the data will be loaded from the test archive + RESULT + ------ + exist: boolean + indicates whether data was found (True) or not (False) + time: list + the timestamps of the profile + s: list + the surface coordinates of the profile in meter + heatflux: list + the heat flux for the profile in W/m2 + + + NOTE + ------ """ goon = True if time_window == None: @@ -3402,7 +5396,7 @@ def extract_temperature_profile_from_DL(port, time, images, profile, finger=None time_window=None, inputchecked=False, verbose=0, reference_time=0): """ - + for a given heatflux image and profile and finger information, a profile can be extracted. INPUT ------ port: int or str @@ -3411,14 +5405,15 @@ def extract_temperature_profile_from_DL(port, time, images, profile, finger=None the time vector for the given images images: numpy array temperature flux images, from which the profile should be extracted - profile: - - finger: - - time_window: - - inputchecked: - + profile: string, integer or float, + string: "TM3h_5_5" or "1605" or "16.05" or "5", similar for float and integer. single number only if the finger is given! + finger: string or integer, + string "TM3h_5" or "16" + time_window: integer, float or list/numpy array + defines the timewindow, if only a number is give or an array/list with only one number, it searches one profile for the given timevector, \n + if a list or array is given, several profiles will be returned, in case of None, the full program will be returned + inputchecked: boolean, optional, default False + defines whether the input parameters have been checked for correct format or not (profile and finger) verbose: integer, optional feedback level (details of print messages) reference_time : float, optional, default is 0 @@ -3426,15 +5421,13 @@ def extract_temperature_profile_from_DL(port, time, images, profile, finger=None RESULT ------ exist: boolean - indicates whether data was found (True) or not (False) - - time - s in m - heat flux in W/m2 - - - - + indicates whether data was found (True) or not (False) + time: list + the timestamps of the profiles + s: list + the surface coordinates of the profiles in meter + heatflux: list + the heat flux for each profile in W/m2 NOTES ----- """ @@ -3496,8 +5489,15 @@ def extract_temperature_profile_from_DL(port, time, images, profile, finger=None def download_divertor_load(port, targetmodule=None, program=None, tstart=None, tend=None, version=0, verbose=0, testmode=False, - request=True): + request=True, request_ALL=False): """ + downloads the integrated divertor load, based on the heat flux evaluation of the temperature data from the IR cameras.\n + The data is structed in total load per divertor, identified by the port, and also in target-modules of the divertor.\n + An estiamted error for the data is given back. + To get the total divertor load, please download the data of each of the 10 AEF ports.\n + In case one or more ports are missing, request can be created (console will give feedback on the request).\n + The data will be evaluated by a server over time. + INPUT ------ port: int or str @@ -3506,21 +5506,30 @@ def download_divertor_load(port, targetmodule=None, program=None, tstart=None, name or description of the target module, nothing wil return total load program: str program id as str in the format yyyymmdd.pid - tstart: uint64 - - tend: uint64 - - version: int - - verbose: int - - testmode: - + tstart: integer + time in ns where the search should start + tend: integer + time in ns where the search should end + version: integer, optional, default 0 + Version of the data in the archiveDB, in case of 0 the highest version will be used + verbose: integer, optional + feedback level (details of print messages) + testmode: boolean, default False + True to load data from testarchive request: boolean, optional, default True + switch to turn on or off the request of data if the data is not available in the database + request_ALL: boolean, optional, default False + switch to turn on the request of data for all AEF ports RESULT ------ exist: boolean - + True if data was found, False if not + time: list + a list containing the time stamp of each frame, time is in nanoseconds UTC + load: list + a list of values, values in the images are in W + error: list + a list of values, the error of each timepoint, values are in W """ TMchadict={ 'ALL':(0,1, 'divertor_total_load'), 'FULL':(0,1, 'divertor_total_load'), @@ -3649,9 +5658,13 @@ def download_divertor_load(port, targetmodule=None, program=None, tstart=None, programid=str(tstart) cam_progs=IR_tools.get_work_list(heatflux_requestlist_path, typ='load') cam_progs_ig,reasons=IR_tools.get_work_list(heatflux_requestlist_path, typ='ignore') - if (programid, str(port)) not in cam_progs and (programid, str(port)) not in cam_progs_ig: - f = open(heatflux_requestlist_path+str(now.year)+str(now.month)+"_load_requests.txt", 'a') - f.write(programid+"\t"+str(port)+"\n") + cam_p = cam_progs+cam_progs_ig + if (programid, str(port)) not in cam_p and (programid, "ALL") not in cam_p: + f = open(heatflux_requestlist_path+"Auto_load_requests.txt", 'a') + if request_ALL: + f.write(programid+"\tALL\n") + else: + f.write(programid+"\t"+str(port)+"\n") f.close() if verbose > 0: print(now, "download_divertor_load: request created") @@ -3676,16 +5689,17 @@ def download_scene_model(port, program=None, timepoint=None, version=0, testmode timepoint: int64, optional nanosecond timestamp to indicate the validation window of the scene model version: integer, optional + Version of the data in the archiveDB, in case of 0 the highest version will be used testmode: boolean, optional - verbose: integer, optional + if True, the data will be loaded from the test archive + verbose: integer, optional + feedback level (details of print messages) RESULT ------ exist: boolean indicates whether data was found (True) or not (False) scene_model: dict - dictionary of the different images in the scene model (see keys for describtion) - - + dictionary of the different images in the scene model (see keys for describtion) NOTES ----- @@ -3704,7 +5718,9 @@ def download_scene_model(port, program=None, timepoint=None, version=0, testmode # else: exist, _, timepoint, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if not exist: - raise Exception("download_scene_model: Error! Program "+program+" not found!") +# raise Exception("download_scene_model: Error! Program "+program+" not found!") + logging.warning("download_scene_model: Error! Program "+program+" not found!") + return False, 0 if testmode: base=testarchivepath else: @@ -3745,6 +5761,59 @@ def download_scene_model(port, program=None, timepoint=None, version=0, testmode return exist, scene_model else: return False, [0] +#%% caching, requested by Lukas R, sample function supplied by Lukas R. +import os +def extract_heatflux_profile_from_cache(port, program, time_window=0, profile="TM3h_5_5", **kwargs): + """Returns exists, time, position, heat_flux_profiles + """ + kwargs.setdefault('verbose',0) + kwargs.setdefault('testmode',False) + data_folder = os.path.join(os.path.dirname(__file__), 'cache') + save_path = data_folder + '/'.join(["", program, str(port), ""]) + start = time_window[0] if hasattr(time_window, '__iter__') else 0.0 + end = time_window[-1] if hasattr(time_window, '__iter__') else time_window + path = save_path + "heatflux_images_[{s:.2f}-{e:.2f}].npz".format(s=start, e=end) + if not os.path.isfile(path): + if kwargs['verbose'] > 0: + print("Generating cache for {program} port {port} timewindow [{s:.2f}-{e:.2f}]".format( + port=port, program=program, s=start, e=end + )) + cache_heatflux_by_program(port, program, time_window=time_window, **kwargs) + cache_file = np.load(path) + found,time,images = [cache_file[kw] for kw in ['found','time','images']] + if not found: + raise ValueError("Download was invalid") + exist, _, t1, t6 = get_trigger_from_PID( + program, port, kwargs['testmode'], verbose=kwargs['verbose']-1) + return extract_heatflux_profile_from_DL((np.asarray(time)-t1)/1e9, images, profile, time_window=time_window, **kwargs) + +def cache_heatflux_by_program(port, program, time_window=0, **kwargs): + """ + """ + data_folder = os.path.join(os.path.dirname(__file__), 'cache') + save_path = data_folder + '/'.join(["", program, str(port), ""]) + check_path_and_make_folders(True, save_path) + found, time, images = download_heatflux_by_program(port, program, + time_window=time_window, **kwargs) + start = time_window[0] if hasattr(time_window, '__iter__') else 0.0 + end = time_window[-1] if hasattr(time_window, '__iter__') else time_window + np.savez(save_path + "heatflux_images_[{s:.2f}-{e:.2f}]".format(s=start, e=end), + found=found, time=time, images=images) + +def check_path_and_make_folders(activate, path, sep='', folder=''): + '''Does nothing if file structure already exists. + Otherwise recursively goes up layers until existing file is found, then creates subfolders. + This kind of thing should be in an independent & general caching handler + ''' + if not activate: + return + if not os.path.exists(path): + check_path_and_make_folders(activate, *path.rpartition('/')) + try: + os.mkdir(path+sep+folder) + except FileExistsError: + pass + #%% general download functions """ @@ -3757,7 +5826,8 @@ def read_restdb_old(request_url): Reads JSON data from W7-X REST API INPUT ------ - + request_url: string + the url from there the data should be loaded Returns ------ valid: bolean @@ -3788,7 +5858,10 @@ def read_restdb_old(request_url): def read_restdb(request_url): """ Reads JSON data from W7-X REST API. Signals are converted into double - + INPUT + ------ + request_url: string + the url from there the data should be loaded Returns: ------ valid: bolean diff --git a/plot_IR_data.py b/plot_IR_data.py index 1fab6535eff18ca4017a05a96f906be0f897453b..7f552b86bcd573df6f62f90a90dd2c3a74d3355e 100644 --- a/plot_IR_data.py +++ b/plot_IR_data.py @@ -16,8 +16,8 @@ if __name__ == '__main__': #%% loading data print(datetime.datetime.now(), "start") - status, time, images, valid = downIR.get_temp_from_raw_by_program(10, - "20180920.017", + status, time, images, valid = downIR.get_temp_from_raw_by_program(51, + "20171206.045", time_window=[1, 1.1], emi=0.8, T_version=2, diff --git a/plot_heatflux_example.py b/plot_heatflux_example.py index 88c6a82619ae708bcc851f82b42e10474327e308..f86f3047ae7902995b77d5b9253d06a2e7d94831 100644 --- a/plot_heatflux_example.py +++ b/plot_heatflux_example.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ Created on Thu Nov 29 17:41:40 2018 -V3.2.0 +V3.4.1 @author: holn """ import numpy as np @@ -18,7 +18,7 @@ if __name__=='__main__': #%% loading data print(datetime.datetime.now(),"start") - status,times,images=IR.download_heatflux_by_program(port,program,time_window=[0,2],version=2,threads=1,verbose=5,testmode=False) + status,times,images=IR.download_heatflux_by_program(port,program,time_window=[0,2],version=2,verbose=5,testmode=False) print('done') #%% plotting data diff --git a/setup.py b/setup.py index 3c465220c45a1572b02efa71174eae46900befb8..9bde9f6074bfb0fb77ff2a31a305fc5fb95e7ef5 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name = 'ir-data-access', - version = '3.3.2', + version = '3.4.0', author = 'Holger Niemann, Peter Drewelow', author_email = 'holger.niemann@ipp.mpg.de', description = 'Access Frontend for IR camera data', @@ -16,7 +16,7 @@ setup( 'plot_heatflux_example' ], data_files=[ - ('',['upload_config','CHANGELOG','ToDo.txt']), + ('',['upload_config','CHANGELOG','ToDo.txt','README.md']), ('data',['data/AEF10_coldframes_background_fails_real.txt', 'data/AEF11_coldframes_background_fails_real.txt', 'data/AEF20_coldframes_background_fails_real.txt', @@ -30,7 +30,7 @@ setup( 'data/finger_info_TDU.csv']) ], install_requires = [ - 'archivedb>=0.2.0' + 'w7xarchive>=0.11.21' ] );