diff --git a/downloadversionIRdata.py b/downloadversionIRdata.py index 809d3b9909916f1fe47780f90ce28105bd7ad177..797c92f4562c5eef47ca7fef988471bc3cae8c1f 100644 --- a/downloadversionIRdata.py +++ b/downloadversionIRdata.py @@ -4,18 +4,22 @@ Created on Wed Oct 25 15:51:46 2017 updated on Tue Aug 21 10:20:00 2018 last update on Fr Nov 23 15:37:00 2018 -Version: 3.3.2 -(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes ) +Version: 3.3.3 +(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) + . + #of updates to add functionalities + . + #number of updates for bug fixes ) @author: holn """ import numpy as np import IR_image_tools as IR_tools -from IR_config_constants import archivepath,testarchivepath,portcamdict,camlimdict, \ - valid_FOV_circle,valid_background_rectangle, \ - TC_port,TC_channel,IRCamColdframes_fittingpath, \ - exJet,portpathdict,project,project_ana, \ - heatflux_requestlist_path#,exJet_trans +from IR_config_constants import archivepath, testarchivepath, portcamdict, camlimdict, \ + valid_FOV_circle, valid_background_rectangle, \ + TC_port, TC_channel, IRCamColdframes_fittingpath, \ + exJet, portpathdict, project, project_ana, \ + heatflux_requestlist_path#, exJet_trans #try: # import W7Xrest.read_restdb as AKF_1 #except: @@ -29,164 +33,187 @@ from PIL import Image from io import BytesIO from os.path import join import matplotlib.pyplot as plt -try: +try: import threading import archivedb as AKF_2 # disbale deprecation warnings, because of timezone warning spam import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) - fastDL=True + fastDL = True except Exception as E: print(E) - fastDL=False + fastDL = False try: - config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\" - FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0] + config_path = "\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\"\ + "Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\" + FOV = plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:, :, 0] del FOV except: - config_path="" + config_path = "" -def get_latest_version(stream,project="W7X",testmode=False,t_from=None,t_to=None, - program=None,verbose=0): - - """Find out latest version of given stream - :param stream url stream of interest - :return int of version number of None if non-versioned or non-existing stream +def get_latest_version(stream, project="W7X", testmode=False, t_from=None, + t_to=None, program=None, verbose=0): + """Find out latest version of given stream + + INPUT + ------ + stream: string + url stream of interest + project: string, optional, default 'W7X' + the main tree in the database + testmode: boolean, optional, default False + whether ArchiveDB oder Test Archive should be used for the request + t_from: uint64, optional + timestamp in nanosecond from where the data version should be identified + t_to: uint64, optional + timestamp in nanosecond up to where the data version should be identified + program: string, optional + program id as a string in the form of "yyyymmdd.pid" + verbose: integer, optional, default=0 + defines the output level, 1 only this function, 2 this and one level below functions etc. + RESULT + ------ + version: integer or None + return int of version number of None if non-versioned or non-existing stream author: G. Schlisio, edit by holn """ - if testmode: - base=testarchivepath - else: - base=archivepath - - if program==None and t_from==None and t_to==None: - request =urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"}) - else: - if program is not None: + if testmode: + base = testarchivepath + else: + base = archivepath + + if program == None and t_from == None and t_to == None: + request = urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"}) + else: + if program is not None: # prog=get_program_from_PID(program) # if prog[0]: # t_from=prog[1]['trigger']['0'][0] # t_to=prog[1]['trigger']['6'][0] - exist, t_from, _, t_to = get_trigger_from_PID(program, stream, testmode, verbose=verbose-1) - - elif t_from is not None and t_to==None: - t_to=int(t_from+100e9) - elif t_from==None and t_to is not None: - t_from=int(t_to-10e9) - request =urllib.request.Request(base +project+"/" + stream + "/_versions.json?from="+str(t_from)+"&upto="+str(t_to), headers={"Accept": "application/json"}) - try: - response = urllib.request.urlopen(request) - d = json.loads(response.read().decode('utf-8')) - except urllib.error.HTTPError as ex: - msg = ex.read() - raise RuntimeError(msg) - except Exception as E: - raise RuntimeError(E) - else: - response.close() + exist, t_from, _, t_to = get_trigger_from_PID(program, None, testmode, verbose=verbose-1) + if not exist: + raise Exception("Cannot identify the Program") + + elif t_from is not None and t_to == None: + t_to = int(t_from+100e9) + elif t_from == None and t_to is not None: + t_from = int(t_to-10e9) + request = urllib.request.Request(base +project+"/" + stream + "/_versions.json?from="+str(t_from)+"&upto="+str(t_to), headers={"Accept": "application/json"}) + try: + response = urllib.request.urlopen(request) + d = json.loads(response.read().decode('utf-8')) + except urllib.error.HTTPError as ex: + msg = ex.read() + raise RuntimeError(msg) + except Exception as E: + raise RuntimeError(E) + else: + response.close() # detect unversioned or non-existing stream - if d["versionInfo"] == []: - return None - versions = [] - for i in d['versionInfo']: - versions.append(i['number']) + if d["versionInfo"] == []: + return None + versions = [] + for i in d['versionInfo']: + versions.append(i['number']) - return max(versions) + return max(versions) -def TimeToNs(date,time): +def TimeToNs(date, time): """ - TimeToNs(date,time) - (list) date [year,month,day] - (list) time [hours,minutes,seconds,microseconds] - """ - date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3]) - div=date_time-datetime.datetime(1970,1,1,0,0,0) - nsdate=div.total_seconds()*1e9 + TimeToNs(date, time) + (list) date [year, month, day] + (list) time [hours, minutes, seconds, microseconds] + """ + date_time = datetime.datetime(date[0], date[1], date[2], time[0], time[1], + time[2], time[3]) + div = date_time-datetime.datetime(1970, 1, 1, 0, 0, 0) + nsdate = div.total_seconds()*1e9 return nsdate -def read_program(timestamp_start,timestamp_end=0,tol=60): +def read_program(timestamp_start, timestamp_end=0, tol=60): """ read_program() """ program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from' - if timestamp_end==0: + if timestamp_end == 0: timestamp_end = timestamp_start jsonurl = '{0}={1}&upto={2}'.format(program_url, int(timestamp_start-tol*1e9), int(timestamp_end+tol*1e9)) try: res = urllib.request.urlopen(jsonurl) - prog_raw=res.read() + prog_raw = res.read() res.close() except urllib.error.URLError as e: print('read_program: Error opening URL') print(e) - return False,0 + return False, 0 except Exception as e: - print('read_program: Error opening URL',e) - return False,0 + print('read_program: Error opening URL', e) + return False, 0 else: - prog_string=prog_raw.decode(encoding='UTF-8') + prog_string = prog_raw.decode(encoding='UTF-8') prog_list = json.loads(prog_string) - pl=prog_list['programs'][0] + pl = prog_list['programs'][0] return True, pl -def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, - testmode=False,verbose=0): +def download_LUT(port, time, exposure=0, emissivity=0, camera_filter=0, version=0, + testmode=False, verbose=0): """ - download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1): + download_LUT(camera, port, time, exposure=0, emissivity=0, camera_filter=0, version=1): time in ns Have to swap 11, 21 until correction in the database - """ - OP=IR_tools.get_OP_by_time(time_ns=time) + """ + OP = IR_tools.get_OP_by_time(time_ns=time) if port == 21: port = 11 elif port == 11: port = 21 if testmode: - larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_" + larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_" else: - larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_" - if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec": - query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity)) - elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam": - query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity)) + larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_" + if port == 50 and OP == "OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec": + query = "Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity)) + elif port in [10, 11, 20, 21, 30, 31, 40, 41, 51] or OP == "OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam": + query = "Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity)) else: print("download_LUT: Error! Camera unknown, stopping here.") raise Exception - if version==0: - version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM",t_from=time,testmode=testmode) - if verbose>0: + if version == 0: + version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM", t_from=time, testmode=testmode) + if verbose > 0: print("download_LUT: LUT V"+str(version)+" is used") - #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0])) - LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20)) + #time=int(fu.TimeToNs([2017,9,26], [8, 0, 0, 0])) + LUTpar = read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20)) if LUTpar[0]: - LUTid=LUTpar[2][0]['structure'][query] - LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20)) + LUTid = LUTpar[2][0]['structure'][query] + LUTs = read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20)) if LUTs[0]: - LUTs=LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back - LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]] + LUTs = LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back + LUT = [LUTs[0], LUTs[LUTid], LUTs[LUTid+1]] del LUTpar, LUTs - return True,LUT + return True, LUT else: print("download_LUT: Warning! unable to download the LUTs") del LUTpar, LUTs - return False,0 + return False, 0 else: del LUTpar print("download_LUT: Warning! unable to find LUTs, check your request") - return False,0 + return False, 0 + -def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0): +def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, verbose=0):#emissivity_steel=0.31, """ read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1) - + Read LUT V3 from local files for testing new calibration. """ LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT' @@ -195,17 +222,17 @@ def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_s filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity) # data = json.loads(join(LUT_dir, filename).decode('utf-8')) try: - with open(LUT_dir+"\\"+filename) as data_file: - jfile=json.load(data_file) - LUT=np.array(jfile['LUT']).swapaxes(0,1) - if verbose>0: + with open(LUT_dir+"\\"+filename) as data_file: + jfile = json.load(data_file) + LUT = np.array(jfile['LUT']).swapaxes(0, 1) + if verbose > 0: print("read_LUT_from_file: succesfully loaded V3 LUT from local directory") return True, LUT except Exception as E: - print("read_LUT_from_file: ERROR in loading V3 LUTs",E) + print("read_LUT_from_file: ERROR in loading V3 LUTs", E) return False, [] - -def download_NUC_by_program(port,program,exposure,version=0,testmode=False,verbose=0): + +def download_NUC_by_program(port, program, exposure, version=0, testmode=False, verbose=0): ''' ''' # prog=get_program_from_PID(program) @@ -217,58 +244,60 @@ def download_NUC_by_program(port,program,exposure,version=0,testmode=False,verbo # stoptime=prog[1]['trigger']['1'][0] exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if exist: - return download_NUC_by_times(port,starttime,stoptime,exposure,version,verbose=verbose) + return download_NUC_by_times(port, starttime, stoptime, exposure, version, verbose=verbose) else: # except: print("download_NUC_by_program: cannot find the program") - return False,0,0 + return False, 0, 0 -def download_NUC_by_times(port,starttime,stoptime,exposure,testmode=False,version=0,verbose=0): +def download_NUC_by_times(port, starttime, stoptime, exposure, testmode=False, version=0, verbose=0): ''' ''' - OP=IR_tools.get_OP_by_time(time_ns=starttime) + OP = IR_tools.get_OP_by_time(time_ns=starttime) if testmode: - larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_" + larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_" else: - larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_" + larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_" # NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) - if version==0: - version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM",t_from=starttime,testmode=testmode) + if version == 0: + version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM", t_from=starttime, testmode=testmode) try: res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) signal_list = json.loads(res.read().decode('utf-8')) res.close() - goon=True + goon = True except urllib.error.URLError as e: print('download_NUC_by_times: Error! ', e) - goon=False + goon = False except Exception as e: print('download_NUC_by_times: Error! ', e) - goon=False - if goon: - n=0 - nuctimes=[0] + goon = False + if goon: + n = 0 + nuctimes = [0] for NUCpar in signal_list['values']: - if NUCpar['structure']['Texp']==exposure: - nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]] + if NUCpar['structure']['Texp'] == exposure: + nuctimes = [signal_list['dimensions'][2*n], signal_list['dimensions'][2*n+1]] # gain_i=NUCpar['structure']['gain_index'] - offset_i=NUCpar['structure']['offset_index'] - n+=1 - if nuctimes[0]!=0: - NUC=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10)) + offset_i = NUCpar['structure']['offset_index'] + n += 1 + if nuctimes[0] != 0: + NUC = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10)) if NUC[0]: - images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i) - return True,images,['gain','offset','cold','badpixels','gain_error','offset_error'] + images = np.vsplit(NUC[2], np.shape(NUC[2])[0]/offset_i) + return True, images, ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error'] else: - print("download_NUC_by_times: NUC image for requested exposure time not found") + if verbose > 0: + print("download_NUC_by_times: NUC image for requested exposure time not found") return False, 0, 0 else: - print("download_NUC_by_times: NUC image for requested exposure time not found") - return False,0,0 + if verbose > 0: + print("download_NUC_by_times: NUC image for requested exposure time not found") + return False, 0, 0 else: - return False,0,0 + return False, 0, 0 -def get_NUC_by_program(port,program,exposure,version=0,testmode=False,verbose=0): +def get_NUC_by_program(port, program, exposure, version=0, testmode=False, verbose=0): ''' ''' # prog=get_program_from_PID(program) @@ -280,239 +309,240 @@ def get_NUC_by_program(port,program,exposure,version=0,testmode=False,verbose=0) # stoptime=prog[1]['trigger']['1'][0] exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if exist: - return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,testmode=testmode,verbose=verbose) + return get_NUC_by_times(port, starttime, stoptime, exposure, version=version, testmode=testmode, verbose=verbose) else: # except: print("get_NUC_by_program: Warning! cannot find the trigger timestamps for program", program) -# return False,0,0 - OP=IR_tools.get_OP_by_time(program_str=program) - return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,testmode=testmode,OP=OP,verbose=verbose) - -def get_NUC_by_times(port,starttime,stoptime,t_exp,version=0,testmode=False,OP=None,verbose=0): +# return False, 0, 0 + OP = IR_tools.get_OP_by_time(program_str=program) + return get_NUC_by_times(port, starttime, stoptime, exposure, version=version, testmode=testmode, OP=OP, verbose=verbose) + +def get_NUC_by_times(port, starttime, stoptime, t_exp, version=0, testmode=False, OP=None, verbose=0): """ Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera in a given port and time interval. Depending on time and camera, reconstruct - the cold frame and/or compute the gain,offset from the hot and cold frame. + the cold frame and/or compute the gain, offset from the hot and cold frame. """ if OP is None: - OP=IR_tools.get_OP_by_time(time_ns=stoptime) - if OP=="OP1.2a": - t1=stoptime - t0=starttime - prog=read_program(t1) - program=prog[1]['id'] + OP = IR_tools.get_OP_by_time(time_ns=stoptime) + if OP == "OP1.2a": + t1 = stoptime + t0 = starttime + prog = read_program(t1) + program = prog[1]['id'] if (port == 31) or (port == 21 and float(program[4:]) > 1110): - if verbose>0: + if verbose > 0: print("get_NUC_by_times: rebuilding coldframe") #use any way the rebuilt coldframe. - sT = get_sensor_temp_by_program(port, program,testmode=testmode)[2][0] + sT = get_sensor_temp_by_program(port, program, testmode=testmode)[2][0] # coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp) - gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp,testmode=False) + gotit, hotcold, describtion = download_hot_cold_reference_by_times(port, t_exp, testmode=False) + del describtion if gotit: - coldref=hotcold[1] - hotref=hotcold[0] + coldref = hotcold[1] + hotref = hotcold[0] else: raise Exception("get_NUC_by_times: unable to download reference frames") filestring = 'AEF' + str(port) + '_et' + str(int(t_exp)) amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy')) bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy')) cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref) - if verbose>0: + if verbose > 0: print('get_NUC_by_times: calculate gain, offset and bad pixels') - gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1) + gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref, verbose=verbose-1) gain[gain == np.inf] = 0 offset[offset == np.inf] = 0 - badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1) - gain_error=0 - offset_error=0 - return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error'] + badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1) + gain_error = 0 + offset_error = 0 + return True, [gain, offset, cirebuild, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error'] elif (port == 11) or (port == 21): - if verbose>0: + if verbose > 0: print('get_NUC_by_times: downloading NUC') # coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp) - gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp,testmode=False) + gotit, hotcold, describtion = download_hot_cold_reference_by_times(port, t_exp, testmode=False) if gotit: - coldref=hotcold[1] - hotref=hotcold[0] + coldref = hotcold[1] + hotref = hotcold[0] else: raise Exception("get_NUC_by_times: unable to download reference frames") - NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version,testmode) - if NUC_DL[0]==False: + NUC_DL = download_NUC_by_times(port, t0, t1, t_exp, version, testmode) + if NUC_DL[0] == False: raise Exception("get_NUC_by_times: NUC was not found") else: ci = NUC_DL[1][2] - gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1) + gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref, verbose=verbose-1) gain[gain == np.inf] = 0 offset[offset == np.inf] = 0 - badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1) - gain_error=0 - offset_error=0 - return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error'] + badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1) + gain_error = 0 + offset_error = 0 + return True, [gain, offset, ci, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error'] else: - if verbose>0: + if verbose > 0: print('get_NUC_by_times: downloading NUC') - return download_NUC_by_times(port,starttime,stoptime,t_exp,testmode=testmode) - elif OP=="OP1.2b": - gain_error=0 - offset_error=0 - gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp,testmode=False) + return download_NUC_by_times(port, starttime, stoptime, t_exp, testmode=testmode) + elif OP == "OP1.2b": + gain_error = 0 + offset_error = 0 + gotit, hotcold, describtion = download_hot_cold_reference_by_times(port, t_exp, testmode=False) if gotit: - coldref=hotcold[1] - hotref=hotcold[0] + coldref = hotcold[1] + hotref = hotcold[0] else: raise Exception("get_NUC_by_times: unable to download reference frames") ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way try: - exist,_,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,stoptime,frametype=0,version=0,testmode=testmode,verbose=verbose-1) + exist, _, coldframes = download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, frametype=0, version=0, testmode=testmode, verbose=verbose-1) except: - exist = False + exist = False if exist: - cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64) + cold = np.zeros(np.shape(coldframes[0]), dtype=np.uint64) for ele in coldframes: - cold+=ele - cold=np.asarray(cold/len(coldframes),dtype=np.uint16) + cold += ele + cold = np.asarray(cold/len(coldframes), dtype=np.uint16) else: print('get_NUC_by_times: Warning! no cold frames found. will use reference cold frame...') cold = np.copy(coldref) -# return False,[0],[0] - gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1) - badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1) - return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error'] +# return False, [0], [0] + gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref, verbose=verbose-1) + badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1) + return True, [gain, offset, cold, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error'] else: raise Exception("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP") -def download_calibration_raw_files_by_time(port,t_exp,starttime,stoptime,frametype=0,version=0,testmode=False,verbose=0): +def download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, frametype=0, version=0, testmode=False, verbose=0): """ - download_calibration_raw_files_by_time(port,t_exp,starttime,stoptime,frametype=0,version=0) + download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, frametype=0, version=0) frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background) """ - gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(stoptime-20e6),testmode=testmode) - OP=IR_tools.get_OP_by_time(time_ns=stoptime) + gotit, time_t, texp_t = get_exposure_by_times(port, int(starttime-100), int(stoptime-20e6), testmode=testmode) + OP = IR_tools.get_OP_by_time(time_ns=stoptime) if gotit: - expinds=np.where(texp_t==t_exp)[0] - if len(expinds)==0: + expinds = np.where(texp_t == t_exp)[0] + if len(expinds) == 0: print("download_calibration_raw_files_by_time: Error! cannot find the exposure time in the given data") - return False,[0],[0] + return False, [0], [0] else: print("download_calibration_raw_files_by_time: Error! exposure time not found") - return False,[0],[0] - gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(stoptime-20e6),testmode=testmode) + return False, [0], [0] + gotitf, timef, values_f = get_frametype_by_times(port, int(starttime-100), int(stoptime-20e6), testmode=testmode) if gotitf: - typiinds=np.where(values_f[expinds]==frametype)[0] - if len(typiinds)>0: - ref_t=[np.min(timef[expinds][typiinds]),np.max(timef[expinds][typiinds])] + typiinds = np.where(values_f[expinds] == frametype)[0] + if len(typiinds) > 0: + ref_t = [np.min(timef[expinds][typiinds]), np.max(timef[expinds][typiinds])] # print((ref_t[1]-ref_t[0])/1e9) -# print(len(timef),len(timef[expinds]),len(timef[expinds][typiinds])) +# print(len(timef), len(timef[expinds]), len(timef[expinds][typiinds])) else:#okay the early data stuff or strange stuff - if verbose>0: + if verbose > 0: print("download_calibration_raw_files_by_time: frame type was not identified assuming that the first part is the cold and the second one the background") - frametimes=time_t[expinds] - diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))] - turnpoint=np.where(np.asarray(diftime)>11e6)[0][0] - if frametype==0: - ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])] + frametimes = time_t[expinds] + diftime = [frametimes[i]-frametimes[i-1] for i in range(1, len(frametimes))] + turnpoint = np.where(np.asarray(diftime) > 11e6)[0][0] + if frametype == 0: + ref_t = [np.min(frametimes[0:turnpoint+1]), np.max(frametimes[0:turnpoint+1])] # print((ref_t[1]-ref_t[0])/1e9) - elif frametype==1: -# print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes)) - ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])] + elif frametype == 1: +# print(len(frametimes[turnpoint+1:]), len(frametimes[0:turnpoint+1]), len(frametimes)) + ref_t = [np.min(frametimes[turnpoint+1:]), np.max(frametimes[turnpoint+1:])] # print((ref_t[1]-ref_t[0])/1e9) else: raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!") else: print("download_calibration_raw_files_by_time: Error! frametype not found") - frametimes=time_t[expinds] - diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))] - turnpoint=np.where(np.asarray(diftime)>11e6)[0][0] - if frametype==0: - ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])] + frametimes = time_t[expinds] + diftime = [frametimes[i]-frametimes[i-1] for i in range(1, len(frametimes))] + turnpoint = np.where(np.asarray(diftime) > 11e6)[0][0] + if frametype == 0: + ref_t = [np.min(frametimes[0:turnpoint+1]), np.max(frametimes[0:turnpoint+1])] print((ref_t[1]-ref_t[0])/1e9) - elif frametype==1: -# print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes)) - ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])] + elif frametype == 1: +# print(len(frametimes[turnpoint+1:]), len(frametimes[0:turnpoint+1]), len(frametimes)) + ref_t = [np.min(frametimes[turnpoint+1:]), np.max(frametimes[turnpoint+1:])] print((ref_t[1]-ref_t[0])/1e9) else: raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!") -# return False,[0],[0] - t1date=datetime.datetime.utcfromtimestamp((stoptime-100)/1e9) - t1date=t1date.isoformat() - t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9) - t0date=t0date.isoformat() - if version==0: - version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime,testmode=testmode) +# return False, [0], [0] + t1date = datetime.datetime.utcfromtimestamp((stoptime-100)/1e9) + t1date = t1date.isoformat() + t0date = datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9) + t0date = t0date.isoformat() + if version == 0: + version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM", t_from=starttime, t_to=stoptime, testmode=testmode) if testmode: - larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" - else: - larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" - timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," ")) - t_offset=timest0[-1][0]-time_t[0] - if t_offset>10e6: - if verbose>0: - print("download_calibration_raw_files_by_time: time offset detected, try to correct this, offset is",t_offset) - else: - t_offset=0 -# print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset) - return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset,testmode=testmode,verbose=verbose-1) - -def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000, - testmode=False,version=0): + larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + else: + larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + timest0 = AKF_2.get_time_intervals(larchivepath, t0date.replace("T", " "), t1date.replace("T", " ")) + t_offset = timest0[-1][0]-time_t[0] + if t_offset > 10e6: + if verbose > 0: + print("download_calibration_raw_files_by_time: time offset detected, try to correct this, offset is", t_offset) + else: + t_offset = 0 +# print("starttime frames:", np.min(timest0), "starttime metachannels:", time_t[0], "offset", t_offset) + return download_raw_images_by_times(port, ref_t[0]+t_offset, ref_t[1]+t_offset, testmode=testmode, verbose=verbose-1) + +def download_hot_cold_reference_by_times(port, exposure, starttime=1503907200000000000, + testmode=False, version=0): """ Loads the most recent hot and cold calibration frames for a starttime. Uses first calibration frames if time is not defined. """ - OP=IR_tools.get_OP_by_time(time_ns=starttime) - if isinstance(port,int): - port="AEF"+str(port) + OP = IR_tools.get_OP_by_time(time_ns=starttime) + if isinstance(port, int): + port = "AEF"+str(port) if testmode: - larchivepath=testarchivepath+project+"/"+portpathdict[OP][port]+"raw_" + larchivepath = testarchivepath+project+"/"+portpathdict[OP][port]+"raw_" else: - larchivepath=archivepath+project+"/"+portpathdict[OP][port]+"raw_" + larchivepath = archivepath+project+"/"+portpathdict[OP][port]+"raw_" # NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) - if version==0: - version=get_latest_version(portpathdict[OP][port]+"raw_DATASTREAM",t_from=starttime) + if version == 0: + version = get_latest_version(portpathdict[OP][port]+"raw_DATASTREAM", t_from=starttime) try: path_string = larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)) res = urllib.request.urlopen(path_string) signal_list = json.loads(res.read().decode('utf-8')) res.close() - goon=True + goon = True except urllib.error.URLError as e: - print('download_hot_cold_reference_by_times: Error! ',e) - goon=False + print('download_hot_cold_reference_by_times: Error! ', e) + goon = False except Exception as e: - print('download_hot_cold_reference_by_times: Error! ',e) - goon=False - if goon: - COLDtime=0 - HOTtime=0 + print('download_hot_cold_reference_by_times: Error! ', e) + goon = False + if goon: + COLDtime = 0 + HOTtime = 0 try: - COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us'] + COLDtime = signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us'] except: print("download_hot_cold_reference_by_times: cold image for requested exposure time not found") try: - HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us'] + HOTtime = signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us'] except: print("download_hot_cold_reference_by_times: hot image for requested exposure time not found") - images=[] - if HOTtime!=0: - HOT=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10)) + images = [] + if HOTtime != 0: + HOT = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10)) if HOT[0]: images.append(HOT[2]) - if COLDtime!=0: - COLD=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10)) + if COLDtime != 0: + COLD = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10)) if COLD[0]: images.append(COLD[2]) - - if HOT[0] and COLD[0] and len(images)==2: - return True,images,['hot','cold'] + + if HOT[0] and COLD[0] and len(images) == 2: + return True, images, ['hot', 'cold'] else: print("download_hot_cold_reference_by_times: Error! hot and cold image for requested exposure time not found") - return False,0,0 + return False, 0, 0 else: - return False,0,0 - -def download_background_by_program(port,program,exposure,camera_filter=0,version=0,testmode=False,verbose=0): + return False, 0, 0 + +def download_background_by_program(port, program, exposure, camera_filter=0, version=0, testmode=False, verbose=0): ''' - returned exist,time,frame + returned exist, time, frame ''' # prog=get_program_from_PID(program) # try: @@ -523,96 +553,96 @@ def download_background_by_program(port,program,exposure,camera_filter=0,version # stoptime=prog[1]['trigger']['1'][0] exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if exist: - return download_background_by_times(port,starttime,stoptime,exposure,camera_filter=camera_filter,version=version) + return download_background_by_times(port, starttime, stoptime, exposure, camera_filter=camera_filter, version=version) else:#except: print("download_background_by_program: Error! cannot find the program {0}".format(program)) - return False,0,0 - -def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0, - version=0,testmode=False,verbose=0): - OP=IR_tools.get_OP_by_time(time_ns=starttime) - if OP=="OP1.2a": - stream=portpathdict[OP]["AEF"+str(port)]+"background_" + return False, 0, 0 + +def download_background_by_times(port, starttime, stoptime, exposure, camera_filter=0, + version=0, testmode=False, verbose=0): + OP = IR_tools.get_OP_by_time(time_ns=starttime) + if OP == "OP1.2a": + stream = portpathdict[OP]["AEF"+str(port)]+"background_" if testmode: - larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_" + larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_" else: - larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_" - if version==0: - version=get_latest_version(stream+"DATASTREAM",t_from=starttime,testmode=testmode) + larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_" + if version == 0: + version = get_latest_version(stream+"DATASTREAM", t_from=starttime, testmode=testmode) try: res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) signal_list = json.loads(res.read().decode('utf-8')) res.close() - goon=True + goon = True except urllib.error.URLError as e: - print('download_background_by_times: Error! ',e) - goon=False + print('download_background_by_times: Error! ', e) + goon = False except Exception as e: - print('download_background_by_times: Error! ',e) - goon=False + print('download_background_by_times: Error! ', e) + goon = False if goon: - n=0 - backtimes=[0] + n = 0 + backtimes = [0] for backpar in signal_list['values']: - if backpar['structure']['Texp']==exposure: - if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec": - if backpar['structure']['filter']==camera_filter: - backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]] + if backpar['structure']['Texp'] == exposure: + if port == 50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec": + if backpar['structure']['filter'] == camera_filter: + backtimes=[signal_list['dimensions'][2*n], signal_list['dimensions'][2*n+1]] else: - backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]] - n+=1 - if backtimes[0]!=0: - backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10)) + backtimes=[signal_list['dimensions'][2*n], signal_list['dimensions'][2*n+1]] + n += 1 + if backtimes[0] != 0: + backdat = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10)) if backdat[0]: - return backdat#[True,backdat[2]] + return backdat#[True, backdat[2]] else: print("download_background_by_times: Error! background image for requested exposure time(, filter) not found") - return False,0,0 + return False, 0, 0 else: print("download_background_by_times: Error! background image for requested exposure time(, filter) not found") - return False,0,0 + return False, 0, 0 else: - return False,0,0 - elif OP=="OP1.2b": - NUC_DL=get_NUC_by_times(port,starttime,stoptime,t_exp=exposure,version=version,testmode=testmode,verbose=verbose-1)#download_NUC_by_times(port,t0,t1,t_exp,version) - if NUC_DL[0]==False: + return False, 0, 0 + elif OP == "OP1.2b": + NUC_DL =get_NUC_by_times(port, starttime, stoptime, t_exp=exposure, version=version, testmode=testmode, verbose=verbose-1)#download_NUC_by_times(port, t0, t1, t_exp, version) + if NUC_DL[0] == False: raise Warning("download_background_by_times: NUC was not found") - return False,0,0 +# return False, 0, 0 else: # extract NUC elements gain = [] offset = [] - gain=np.array(NUC_DL[1][0]) - offset=np.array(NUC_DL[1][1]) + gain = np.array(NUC_DL[1][0]) + offset = np.array(NUC_DL[1][1]) # check quality of NUC elements - gain[np.isnan(gain)]=0 - offset[np.isnan(offset)]=0 - offset[offset<-1e100]=0 - return get_NUCed_background_by_times(port=port,t0=starttime,t1=stoptime,t_exp=exposure,cfilter=camera_filter,gain=gain,offset=offset,version=version,verbose=verbose-1) + gain[np.isnan(gain)] = 0 + offset[np.isnan(offset)] = 0 + offset[offset<-1e100] = 0 + return get_NUCed_background_by_times(port=port, t0=starttime, t1=stoptime, t_exp=exposure, cfilter=camera_filter, gain=gain, offset=offset, version=version, verbose=verbose-1) -def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,testmode=False,plot_it=False,verbose=0): +def get_NUCed_background_by_times(port, t0, t1, t_exp, cfilter, gain, offset, version=0, testmode=False, plot_it=False, verbose=0): """ OP1.2b function """ - exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version,testmode=testmode,verbose=verbose-1) - camera=portcamdict["OP1.2b"]["AEF"+str(port)] - if verbose>50: - print("camera is",camera) + exist, btime, backgroundframes=download_calibration_raw_files_by_time(port, t_exp, t0, t1, frametype=1, version=version, testmode=testmode, verbose=verbose-1) + camera = portcamdict["OP1.2b"]["AEF"+str(port)] + if verbose > 50: + print("camera is", camera) background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64) for frame in backgroundframes: - background+=frame - background=np.asarray(background/len(backgroundframes),dtype=np.uint16) + background += frame + background = np.asarray(background/len(backgroundframes),dtype=np.uint16) if plot_it: plt.figure() - plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500) + plt.imshow(background, vmin=np.median(background)-500, vmax=np.median(background)+500) plt.title("background image unnuced") - if not camera.split("_")[0]=="Infratec": - background=apply_NUC([background],gain,offset)[0] + if not camera.split("_")[0] == "Infratec": + background=apply_NUC([background], gain, offset)[0] if plot_it: plt.figure() - plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500) + plt.imshow(background, vmin=np.median(background)-500, vmax=np.median(background)+500) plt.title("background image nuced") - return exist,btime[0],background + return exist, btime[0], background def get_NUCed_coldframe_by_program(port, program, exposure=None, version=0, @@ -651,6 +681,7 @@ def get_NUCed_coldframe_by_program(port, program, exposure=None, version=0, if exposure is None: try: exist, t, exposure = get_exposure_by_program(port, program, version=version, testmode=testmode) + del t except: print('get_NUCed_coldframe_by_program: ERROR! no exposure time found!') return False, [] @@ -658,10 +689,11 @@ def get_NUCed_coldframe_by_program(port, program, exposure=None, version=0, t_exp = exposure[0] else: t_exp = exposure - if verbose>0: - print('get_NUCed_coldframe_by_program: using exposure time {0}'.format(t_exp)) + if verbose > 0: + print('get_NUCed_coldframe_by_program: using exposure time {0}'.format(t_exp)) try: exist, data, desc = get_NUC_by_program(port, program, t_exp, version=version, testmode=testmode, verbose=verbose-1) + del desc except: print('get_NUCed_coldframe_by_program: ERROR! no cold frame found!') return False, [] @@ -674,7 +706,7 @@ def get_NUCed_coldframe_by_program(port, program, exposure=None, version=0, exist2, data, desc = get_NUC_by_program(port, ref_program, t_exp, version=version, testmode=testmode, verbose=verbose-1) if exist2: coldframe = apply_NUC([cold_raw],data[0],data[1])[0] - if verbose>0: + if verbose > 0: print('get_NUCed_coldframe_by_program: using NUC of program {0}'.format(ref_program)) else: print('get_NUCed_coldframe_by_program: WARNING! no adjacent program found! --> cold frame is not NUCed') @@ -694,13 +726,13 @@ def get_NUCed_coldframe_by_program(port, program, exposure=None, version=0, plt.tight_layout(rect=[0, 0, 1, 0.95]) plt.show() else: - print('get_NUCed_coldframe_by_program: ERROR! No cold frame found for program {0}!'.foramt(program)) + print('get_NUCed_coldframe_by_program: ERROR! No cold frame found for program {0}!'.format(program)) coldframe = [] return exist, coldframe -def download_raw_images_by_program(port,program,time_window=0,version=0,testmode=False,verbose=0): +def download_raw_images_by_program(port, program, time_window=0, version=0, testmode=False, verbose=0): """ """ exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) @@ -719,93 +751,93 @@ def download_raw_images_by_program(port,program,time_window=0,version=0,testmode tstop=time_window[1] if tstop<tstart: raise Exception("download_raw_images_by_program: t_stop before t_start") - return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,testmode=testmode,verbose=verbose) + return download_raw_images_by_times(port, int(starttime+tstart*1e9), int(starttime+tstop*1e9), version, testmode=testmode, verbose=verbose) else: if time_window==0: - return download_raw_images_by_times(port,starttime,stoptime,version,testmode=testmode,verbose=verbose) + return download_raw_images_by_times(port, starttime, stoptime, version, testmode=testmode, verbose=verbose) else: - return download_raw_images_by_times(port,starttime,int(starttime+time_window*1e9),version,testmode=testmode,verbose=verbose) + return download_raw_images_by_times(port, starttime, int(starttime+time_window*1e9), version, testmode=testmode, verbose=verbose) else: print("download_raw_images_by_program: Error! cannot find the program") - return False,0,0 + return False, 0, 0 -def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,testmode=False,verbose=0): +def download_raw_images_by_times(port, starttime, stoptime, version=0, intervalSize=1E9, testmode=False, verbose=0): """ """ - OP=IR_tools.get_OP_by_time(time_ns=starttime) - if version==0: - version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime,testmode=testmode) + OP = IR_tools.get_OP_by_time(time_ns=starttime) + if version == 0: + version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM", t_from=starttime, t_to=stoptime, testmode=testmode) if testmode: - larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" else: - larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" - return download_images_by_times(larchivepath,starttime=starttime,stoptime=stoptime,version=version,intervalSize=intervalSize,verbose=verbose) + larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + return download_images_by_times(larchivepath, starttime=starttime, stoptime=stoptime, intervalSize=intervalSize, verbose=verbose)#version=version, -def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalSize=1E9,verbose=0): +def download_images_by_times(larchivepath, starttime, stoptime, intervalSize=1E9, verbose=0):#version=0, """ """ if '_raw' in larchivepath: - typo=np.uint16 + typo = np.uint16 else: - typo=np.float32 + typo = np.float32 if (stoptime-starttime)/intervalSize>1: - nrinterv=int(np.ceil((stoptime-starttime)/intervalSize)) - if verbose>0: - print("download_images_by_times: time_window to large, splitting {0}s interval into {1} smaller fractions".format((stoptime-starttime)/1e9,nrinterv)) - intervalls=[] + nrinterv = int(np.ceil((stoptime-starttime)/intervalSize)) + if verbose > 0: + print("download_images_by_times: time_window to large, splitting {0}s interval into {1} smaller fractions".format((stoptime-starttime)/1e9, nrinterv)) + intervalls = [] for i in range(nrinterv): intervalls.append(int(starttime-10+i*intervalSize)) intervalls.append(stoptime) - time=-1 - allimages=0 - success=False + time = -1 + allimages = 0 + success = False for i in range(nrinterv): try: url = larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1]) - if verbose>2: + if verbose > 2: print("download_images_by_times: reading from\n {0}".format(url)) res = urllib.request.urlopen(url) signal_list = json.loads(res.read().decode('utf-8')) res.close() - images=[np.array(ele, dtype=typo) for ele in signal_list['values']] + images = [np.array(ele, dtype=typo) for ele in signal_list['values']] # signal=np.array(signal_list['values'],dtype=np.uint16) - t=np.array(signal_list['dimensions']) + t = np.array(signal_list['dimensions']) del signal_list - if i==0: - allimages=images.copy() - time=t + if i == 0: + allimages = images.copy() + time = t else: - allimages=allimages+images#np.append(ressignal,signal,axis=0) - time=np.append(time,t) - del images,t - success=True + allimages = allimages+images#np.append(ressignal, signal,axis=0) + time = np.append(time, t) + del images, t + success = True except urllib.error.URLError as e: - print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e)) + print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i, e)) except Exception as e: - print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e)) - return success,time,allimages + print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i, e)) + return success, time,allimages else: try: - if verbose>100: + if verbose > 100: print(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime)) res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime)) signal_list = json.loads(res.read().decode('utf-8')) res.close() - images=[np.array(ele, dtype=typo) for ele in signal_list['values']] + images = [np.array(ele, dtype=typo) for ele in signal_list['values']] # signal=np.array(signal_list['values'],dtype=np.uint16) - t=np.array(signal_list['dimensions']) + t = np.array(signal_list['dimensions']) del signal_list return True, t, images except urllib.error.URLError as e: - print('download_images_by_times: Error! ',e) + print('download_images_by_times: Error! ', e) return False, 0,-1 except Exception as e: - print('download_images_by_times: Error! ',e) + print('download_images_by_times: Error! ', e) return False, 0,-1 if fastDL: - def download_raw_images_by_program_via_png(port,program,time_window=0,version=0, - threads=1,testmode=False,verbose=0): + def download_raw_images_by_program_via_png(port, program, time_window=0, version=0, + threads=1, testmode=False, verbose=0): """ """ # prog=get_program_from_PID(program) @@ -814,43 +846,43 @@ if fastDL: ## prog =AKF_2.get_program_list(t_program[0], t_program[1]) # if not prog[0]: # print("download_raw_images_by_program_via_png: Error! program not found, cannot download the data") -# return False,[0],[0] +# return False, [0], [0] # starttime=prog[1]['trigger']['1'][0] # stoptime=prog[1]['trigger']['6'][0] trigger_exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if not trigger_exist: print("download_raw_images_by_program_via_png: Error! trigger not found, cannot download the data") - OP=IR_tools.get_OP_by_time(time_ns=starttime) + OP = IR_tools.get_OP_by_time(time_ns=starttime) # get raw data from latest or requested version - if version==0: - version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",testmode=testmode,program=program) + if version == 0: + version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM", testmode=testmode, program=program) if testmode: - larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" else: - larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" - return download_images_by_time_via_png(larchivepath,starttime,stoptime, - time_window=time_window,threads=threads, - version=version,verbose=verbose-1) + return download_images_by_time_via_png(larchivepath, starttime, stoptime, + time_window=time_window, threads=threads, + version=version, verbose=verbose-1) - def download_images_by_time_via_png(larchivepath,starttime,stoptime,time_window=0, - threads=1,version=0,verbose=0,framerate='max'): + def download_images_by_time_via_png(larchivepath, starttime, stoptime, time_window=0, + threads=1, version=0, verbose=0, framerate='max'): """ """ - success=True - if type(time_window)==list: - tstart=time_window[0] - tstop=time_window[1] - if tstop<tstart: + success = True + if type(time_window) == list: + tstart = time_window[0] + tstop = time_window[1] + if tstop < tstart: raise Exception("download_images_by_time_via_png: t_stop before t_start") time_from = (starttime-100)+int(tstart*1E9) - if stoptime<time_from: + if stoptime < time_from: raise Exception("download_images_by_time_via_png: time_window after stoptime") time_to = starttime+int(tstop*1E9) - if stoptime<time_to: + if stoptime < time_to: time_to = stoptime print('download_images_by_time_via_png: Warning! time_window cropped by end of data') # stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart) @@ -861,7 +893,7 @@ if fastDL: # stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9) # stdate=stdate.isoformat() time_from = starttime-100 - if time_window==0: + if time_window == 0: time_to = stoptime # enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9) # enddate=enddate.isoformat() @@ -871,60 +903,60 @@ if fastDL: # enddate=enddate.isoformat() #"2017-11-15 08:00:00" try: -# times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," ")) - times=AKF_2.get_time_intervals(larchivepath,time_from,time_to) +# times=AKF_2.get_time_intervals(larchivepath, stdate.replace("T", " "), enddate.replace("T", " ")) + times = AKF_2.get_time_intervals(larchivepath, time_from, time_to) except Exception as E: -# print('download_images_by_time_via_png: Error loading times from ',larchivepath,stdate.replace("T"," "),enddate.replace("T"," ")) - print('download_images_by_time_via_png: Error loading times from {0} between [{1}, {2}]'.format(larchivepath,time_from,time_to)) +# print('download_images_by_time_via_png: Error loading times from ', larchivepath, stdate.replace("T", " "), enddate.replace("T", " ")) + print('download_images_by_time_via_png: Error loading times from {0} between [{1}, {2}]'.format(larchivepath, time_from, time_to)) raise Warning(E) - return False,[0],[0] +# return False, [0], [0] - if framerate!='max': - if verbose>5: - print("download_images_by_time_via_png: downsampling active; number of original frames:",len(times)) - if type(framerate)==float or type(framerate)==int: + if framerate != 'max': + if verbose > 5: + print("download_images_by_time_via_png: downsampling active; number of original frames:", len(times)) + if type(framerate) == float or type(framerate) == int: realrate=1e9/((times[0]-times[-1])[0]/len(times)) - if framerate<realrate: - steps=int(round(realrate/framerate,0)) - dummy=[] - for i in range(0,len(times),steps): + if framerate < realrate: + steps = int(round(realrate/framerate, 0)) + dummy = [] + for i in range(0, len(times), steps): dummy.append(times[i]) - times=dummy - if verbose>5: - print("download_images_by_time_via_png: downsampling active; number of downsampeld frames:",len(times),steps) - time=[] - images=[] - lnt=len(times) - if threads==1: + times = dummy + if verbose > 5: + print("download_images_by_time_via_png: downsampling active; number of downsampeld frames:", len(times), steps) + time = [] + images = [] + lnt = len(times) + if threads == 1: for i in range(lnt): - ele=times[lnt-1-i] - imag=download_last_raw_image_by_time(larchivepath,ele[0]-10,ele[0]+10) + ele = times[lnt-1-i] + imag = download_last_raw_image_by_time(larchivepath, ele[0]-10, ele[0]+10) if imag[0]: time.append(ele[0]) images.append(np.array(imag[1],dtype=np.uint16)) else: - success=False - return success,np.array(time),images + success = False + return success, np.array(time), images else: - tim=[] + tim = [] for i in range(lnt): tim.append(times[lnt-1-i][0]) - intervalls=[] - intervalSize=int(lnt/threads) + intervalls = [] + intervalSize = int(lnt/threads) for i in range(threads): intervalls.append(int(i*intervalSize)) intervalls.append(lnt) jobs = [] resultdict = [] for i in range(threads): - if verbose>0: - print("download_images_by_time_via_png: Start Thread ",i+1) - TH=download_images_thread(i,larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,version) + if verbose > 0: + print("download_images_by_time_via_png: Start Thread ", i+1) + TH=download_images_thread(i, larchivepath, tim[intervalls[i]:intervalls[i+1]],resultdict, version) jobs.append(TH) TH.start() for p in jobs: p.join() - if verbose>0: + if verbose > 0: print("download_images_by_time_via_png: all threads are done") order=[] for ele in resultdict: @@ -938,11 +970,11 @@ if fastDL: times=times+resultdict[order.index(i)][2] resultdict[order.index(i)]=[] del resultdict - return success,np.array(times),images + return success, np.array(times), images class download_images_thread(threading.Thread): - def __init__(self, threadID, larchivepath,times,resultlist,version=0): + def __init__(self, threadID, larchivepath, times, resultlist, version=0): threading.Thread.__init__(self) self.threadID = threadID self.times = times @@ -954,19 +986,19 @@ if fastDL: time=[] successes=[] for i in self.times: - imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version) + imag=download_last_raw_image_by_time(self.archivepath, i-10, i+10, version=self.version) if imag[0]: images.append(np.array(imag[1],dtype=np.uint16)) time.append(i) successes.append(True) else: successes.append(False) - self.resultlist.append([self.threadID,successes,time,images]) + self.resultlist.append([self.threadID, successes, time, images]) class convert_raw_to_temp_thread(threading.Thread): - def __init__(self,larchivepath,times,resultlist,threadID,version=0,background=0, - LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0, - give_ERROR=False,FOV=0,badpixels=[],verbose=0): + def __init__(self, larchivepath, times,resultlist, threadID, version=0, background=0, + LUT=[[], [], []],refT=28.5, gain=0, offset=0, gain_error=0, offset_error=0, + give_ERROR=False, FOV=0, badpixels=[], verbose=0): threading.Thread.__init__(self) self.threadID = threadID self.times = times @@ -989,7 +1021,7 @@ if fastDL: time=[] successes=[] for i in self.times: - imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version) + imag=download_last_raw_image_by_time(self.archivepath, i-10, i+10, version=self.version) if imag[0]: images.append(np.array(imag[1],dtype=np.uint16)) time.append(i) @@ -997,21 +1029,21 @@ if fastDL: else: successes.append(False) if self.give_ERROR: - sucess,images,error_images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR,verbose=self.verbose-1) + success, images, error_images=apply_calib_on_raw(images, self.background, self.LUT, self.refT, self.gain, self.offset, self.gain_error, self.offset_error, False, self.give_ERROR, verbose=self.verbose-1) else: - success,images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR,verbose=self.verbose-1) + success, images=apply_calib_on_raw(images, self.background, self.LUT, self.refT, self.gain, self.offset, self.gain_error, self.offset_error, False, self.give_ERROR, verbose=self.verbose-1) for i in range(len(images)): images[i]=(images[i]*self.FOV).astype(np.float32) - if self.verbose>0: - print(datetime.datetime.now(),"convert_raw_to_temp_thread: correcting bad pixels") - images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1) - self.resultlist.append([self.threadID,successes,time,images]) + if self.verbose > 0: + print(datetime.datetime.now(), "convert_raw_to_temp_thread: correcting bad pixels") + images=IR_tools.correct_images(images, self.badpixels, verbose=self.verbose-1) + self.resultlist.append([self.threadID, successes, time, images]) class NUC_raw_thread(threading.Thread): - def __init__ (self,larchivepath,times,resultlist,threadID,version=0,background=0, - LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0, - give_ERROR=False,FOV=0,badpixels=[],verbose=0): + def __init__ (self, larchivepath, times,resultlist, threadID, version=0, background=0, + LUT=[[], [], []],refT=28.5, gain=0, offset=0, gain_error=0, offset_error=0, + give_ERROR=False, FOV=0, badpixels=[], verbose=0): threading.Thread.__init__(self) self.threadID = threadID self.times = times @@ -1034,28 +1066,28 @@ if fastDL: time=[] successes=[] for i in self.times: - imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version) + imag=download_last_raw_image_by_time(self.archivepath, i-10, i+10, version=self.version) if imag[0]: images.append(np.array(imag[1],dtype=np.uint16)) time.append(i) successes.append(True) else: successes.append(False) - if self.verbose>0: - print(datetime.datetime.now(),"NUC_raw_thread: applying NUC") + if self.verbose > 0: + print(datetime.datetime.now(), "NUC_raw_thread: applying NUC") if self.give_ERROR: - images,error_images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR) + images, error_images=apply_NUC(images, self.gain, self.offset, self.gain_error, self.offset_error, self.give_ERROR) else: - images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR) + images=apply_NUC(images, self.gain, self.offset, self.gain_error, self.offset_error, self.give_ERROR) for i in range(len(images)): images[i]=(images[i]*self.FOV).astype(np.float32) - if self.verbose>0: - print(datetime.datetime.now(),"NUC_raw_thread: correcting bad pixels") - images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1) - self.resultlist.append([self.threadID,successes,time,images]) + if self.verbose > 0: + print(datetime.datetime.now(), "NUC_raw_thread: correcting bad pixels") + images=IR_tools.correct_images(images, self.badpixels, verbose=self.verbose-1) + self.resultlist.append([self.threadID, successes, time, images]) -def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,verbose=0,testmode=True): +def download_raw_FLIR_images_via_png(t1, t6, time_window=0, version=0, threads=1, verbose=0, testmode=True): """ """ if testmode: @@ -1089,18 +1121,18 @@ def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,ver # enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_window) # enddate=enddate.isoformat() #"2017-11-15 08:00:00" - typ1_version=get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM",testmode=testmode,t_from=tstart) + typ1_version = get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM", testmode=testmode, t_from=tstart) if typ1_version==None: mode=1 else: mode=2 if mode==1: ### okay the data is stored probably as different sets (so recorded with FLIR software) - timess,versions=get_FLIR_Pset_times(starttime,stoptime,version,testmode=testmode)#stdate,enddate,version,testmode=testmode) - if verbose>0: + timess, versions=get_FLIR_Pset_times(starttime, stoptime, version, testmode=testmode)#stdate, enddate, version, testmode=testmode) + if verbose > 0: print("download_raw_FLIR_images_via_png: found {0} frames in the different sets".format([np.shape(x) for x in timess])) ### we not know the times for all presets for the program, now we have to download them, if times is not empty - exist,time,frames,tsets=download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=verbose-1) + exist, time, frames, tsets=download_FLIR_Psets(larchivepath, timess, threads, versions, verbose=verbose-1) elif mode==2:##the data is stored only in one datastream, at least this datastream exist raise Exception("download_raw_FLIR_images_via_png: not implemented") #download data and downlaod LUT @@ -1111,36 +1143,36 @@ def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,ver # if threads==1: # for i in range(lnt): # ele=times[lnt-1-i] -# imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10) +# imag=download_last_raw_image_by_time(port, ele[0]-10, ele[0]+10) # if imag[0]: # time.append(ele[0]) # images.append(np.array(imag[1],dtype=np.uint16)) # else: # success=False -# return success,np.array(time),images +# return success, np.array(time), images else: raise Exception("download_raw_FLIR_images_via_png: cannot identifie the way the searched data is stored!") - return exist,time,frames,mode,tsets,versions + return exist, time, frames,mode, tsets, versions -def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0,testmode=True): +def get_FLIR_Pset_times(stdate, enddate, version=0, verbose=0, testmode=True): """ """ if testmode: larchivepath=testarchivepath+project+"/QSR07_FLIR/AEK51_raw" else: larchivepath=archivepath+project+"/QSR07_FLIR/AEK51_raw" - if version==0: + if version == 0: versions=[] - versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS0_DATASTREAM",testmode=testmode,t_from=stdate)) - versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS1_DATASTREAM",testmode=testmode,t_from=stdate)) - versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS2_DATASTREAM",testmode=testmode,t_from=stdate)) - versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS3_DATASTREAM",testmode=testmode,t_from=stdate)) + versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS0_DATASTREAM", testmode=testmode, t_from=stdate)) + versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS1_DATASTREAM", testmode=testmode, t_from=stdate)) + versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS2_DATASTREAM", testmode=testmode, t_from=stdate)) + versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS3_DATASTREAM", testmode=testmode, t_from=stdate)) else: - versions=[version,version,version,version] + versions=[version, version, version, version] timess=[] - if version==0: - if verbose>0: + if version == 0: + if verbose > 0: print('get_FLIR_Pset_times: no version specified search for data in all versions') counter=-1 for version0 in versions: @@ -1151,11 +1183,11 @@ def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0,testmode=True): found=False while (version0>0 and not found): try: - url = '{0}_PS{1}_DATASTREAM/V{2}/0/raw'.format(larchivepath,counter,version0) - times_0=AKF_2.get_time_intervals(url,stdate,enddate)#stdate.replace("T"," "),enddate.replace("T"," "))# + url = '{0}_PS{1}_DATASTREAM/V{2}/0/raw'.format(larchivepath, counter, version0) + times_0=AKF_2.get_time_intervals(url, stdate, enddate)#stdate.replace("T", " "), enddate.replace("T", " "))# found=True except Exception as E: - print('get_FLIR_Pset_times: Error querrying {0}\n{1}'.format(url,E)) + print('get_FLIR_Pset_times: Error querrying {0}\n{1}'.format(url, E)) found=False times_0=[] version0=version0-1 @@ -1164,32 +1196,32 @@ def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0,testmode=True): timess.append([]) except: timess.append([]) - if verbose>0: + if verbose > 0: print("get_FLIr_Pset_times: no version for PS"+str(counter)) else: counter=-1 for version0 in versions: counter+=1 try: - url = '{0}_PS{1}_DATASTREAM/V{2}/0/raw'.format(larchivepath,counter,version0) - times_0=AKF_2.get_time_intervals(url,stdate.replace("T"," "),enddate.replace("T"," "))# + url = '{0}_PS{1}_DATASTREAM/V{2}/0/raw'.format(larchivepath, counter, version0) + times_0=AKF_2.get_time_intervals(url, stdate.replace("T", " "), enddate.replace("T", " "))# except Exception as E: - print('get_FLIR_Pset_times: Error querrying {0}\n{1}'.format(url,E)) + print('get_FLIR_Pset_times: Error querrying {0}\n{1}'.format(url, E)) times_0=[] timess.append(times_0) - return timess,versions + return timess, versions -def download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=0): +def download_FLIR_Psets(larchivepath, timess, threads, versions, verbose=0): """ """ if len(timess[0])==len(timess[1]) and len(timess[2])==len(timess[3]) and len(timess[0])==0: - exist=False - time=[] - frames=[] - tsets=[] + exist=False + time=[] + frames=[] + tsets=[] else:#timess is not empty, there is data in at least one Pset - if verbose>0: - print(datetime.datetime.now(),"download_FLIR_Psets: data found, start the download") + if verbose > 0: + print(datetime.datetime.now(), "download_FLIR_Psets: data found, start the download") exist=True timesets=[] framess=[] @@ -1208,56 +1240,56 @@ def download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=0): lnt=len(times) for i in range(lnt): ele=times[lnt-1-i] - imag=download_last_raw_image_by_time(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw",ele[0]-10,ele[0]+10) + imag=download_last_raw_image_by_time(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw", ele[0]-10, ele[0]+10) if imag[0]: - timesets[counter].append([ele[0],i,setn,counter]) + timesets[counter].append([ele[0], i, setn, counter]) framess[counter].append(np.array(imag[1],dtype=np.uint16)) else: success=False else:#several threads, each for a preset - tim=[] - counter=-1 - setn=-1 - threads=0 - larchivepaths=[] - versionss=[] + tim = [] + counter = -1 + setn = -1 + threads = 0 + larchivepaths = [] + versionss = [] for times in timess: - times=np.array(times) - setn+=1 - if times!=[]: - counter+=1 - threads+=1 + times = np.array(times) + setn += 1 + if times != []: + counter += 1 + threads += 1 Psets.append(counter) larchivepaths.append(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw") - tim.append(times[:,0]) + tim.append(times[:, 0]) versionss.append(versions[setn]) jobs = [] resultdict = [] for i in range(threads): - if verbose>0: - print("download_FLIR_Psets: Start Thread ",i+1) - TH=download_images_thread(i,larchivepaths[i],tim[i],resultdict,versionss[i]) + if verbose > 0: + print("download_FLIR_Psets: Start Thread ", i+1) + TH = download_images_thread(i, larchivepaths[i], tim[i],resultdict, versionss[i]) jobs.append(TH) TH.start() for p in jobs: p.join() - if verbose>0: + if verbose > 0: print("download_FLIR_Psets: all threads are done") order=[] for ele in resultdict: order.append(ele[0]) if len(np.where(np.asarray(ele[1])==False)[0])>0: - success=False + success = False for i in range(threads): framess.append(resultdict[order.index(i)][3]) timeline=[] for j in range(len(resultdict[order.index(i)][2])): - timeline.append([resultdict[order.index(i)][2][j],j,Psets[i],i]) + timeline.append([resultdict[order.index(i)][2][j],j, Psets[i], i]) timesets.append(timeline) - resultdict[order.index(i)]=[] - del resultdict,timeline - if verbose>0: - print(datetime.datetime.now(),"download finished, successfull",success) + resultdict[order.index(i)] = [] + del resultdict, timeline + if verbose > 0: + print(datetime.datetime.now(), "download finished, successfull", success) if success: del timess ### okay the have now the data from all presets, now we have to sort the frames into one array @@ -1272,19 +1304,19 @@ def download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=0): time.append(ele[0]) frames.append(framess[ele[2]][ele[1]]) tsets.append(ele[2]) - del timesets,framess + del timesets, framess else: exist=False time=[] frames=[] tsets=[] - return exist,time,frames,tsets + return exist, time, frames, tsets -def download_last_raw_image_by_time(larchivepath,starttime,stoptime,version=0): +def download_last_raw_image_by_time(larchivepath, starttime, stoptime, version=0): """ """ try: - url='{0}/_signal.png?from={1}&upto={2}'.format(larchivepath,starttime-10,stoptime) + url='{0}/_signal.png?from={1}&upto={2}'.format(larchivepath, starttime-10, stoptime) res = urllib.request.urlopen(url) img = Image.open(BytesIO(res.read())) res.close() @@ -1292,13 +1324,13 @@ def download_last_raw_image_by_time(larchivepath,starttime,stoptime,version=0): pixelarray = np.array(img,dtype=np.uint16)#.swapaxes(0,1) return True, pixelarray except urllib.error.URLError as e: - print('download_last_raw_image_by_time: Error querrying {0}\n{1}'.format(url,e)) + print('download_last_raw_image_by_time: Error querrying {0}\n{1}'.format(url, e)) return False, [-1] except Exception as e: - print('download_last_raw_image_by_time: Error querrying {0}\n{1}'.format(url,e)) + print('download_last_raw_image_by_time: Error querrying {0}\n{1}'.format(url, e)) return False, [-1] -def download_raw_parlog_by_program(port,program,version=0,testmode=False,verbose=0): +def download_raw_parlog_by_program(port, program, version=0, testmode=False, verbose=0): """ """ ## prog=AKF_1.get_program_from_PID(program) @@ -1310,17 +1342,17 @@ def download_raw_parlog_by_program(port,program,version=0,testmode=False,verbose # stoptime=prog[0]['trigger']['6'][0] exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if exist: - return download_raw_parlog_by_times(port,starttime,stoptime,version,testmode=testmode) + return download_raw_parlog_by_times(port, starttime, stoptime, version, testmode=testmode) else: #except: print("download_raw_parlog_by_program: Error! cannot find the program") - return False,0,0 + return False, 0, 0 -def download_raw_parlog_by_times(port,starttime,stoptime,version=0,testmode=False): +def download_raw_parlog_by_times(port, starttime, stoptime, version=0, testmode=False): """ """ - OP=IR_tools.get_OP_by_time(time_ns=stoptime) - if version==0: - version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG",t_from=starttime,testmode=testmode) + OP = IR_tools.get_OP_by_time(time_ns=stoptime) + if version == 0: + version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG", t_from=starttime, testmode=testmode) if testmode: larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version) else: @@ -1331,16 +1363,18 @@ def download_raw_parlog_by_times(port,starttime,stoptime,version=0,testmode=Fals res.close() goon=True except urllib.error.URLError as e: - print('download_raw_parlog_by_times: Error! ',e) + print('download_raw_parlog_by_times: Error! ', e) goon=False except Exception as E: - print('download_raw_parlog_by_times: Error! ',E) + print('download_raw_parlog_by_times: Error! ', E) goon=False if goon: - return True, signal_list['dimensions'],signal_list['values'][0]['meta-data'] + return True, signal_list['dimensions'], signal_list['values'][0]['meta-data'] + else: + return False, [0], [0] -def get_INFRATEC_filter_by_program(program,port=50,version=0,testmode=False,verbose=0): +def get_INFRATEC_filter_by_program(program, port=50, version=0, testmode=False, verbose=0): """ """ # prog=get_program_from_PID(program) @@ -1352,17 +1386,17 @@ def get_INFRATEC_filter_by_program(program,port=50,version=0,testmode=False,verb # stoptime=prog[1]['trigger']['6'][0] exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if exist: - return get_INFRATEC_filter_by_times(starttime,stoptime,50,version,testmode=testmode) + return get_INFRATEC_filter_by_times(starttime, stoptime,50, version, testmode=testmode) else:#except: print("get_INFRATEC_filter_by_program: Error! cannot find the program") - return False,0,0 + return False, 0, 0 -def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0,testmode=False): +def get_INFRATEC_filter_by_times(starttime, stoptime, port=50, version=0, testmode=False): """ """ - OP=IR_tools.get_OP_by_time(time_ns=starttime) - if version==0: - version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG",t_from=starttime,testmode=testmode) + OP = IR_tools.get_OP_by_time(time_ns=starttime) + if version == 0: + version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG", t_from=starttime, testmode=testmode) if testmode: larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version) @@ -1374,17 +1408,17 @@ def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0,testmode=F res.close() goon=True except urllib.error.URLError as e: - print('get_INFRATEC_filter_by_times: Error! ',e) + print('get_INFRATEC_filter_by_times: Error! ', e) goon=False except Exception as E: - print('get_INFRATEC_filter_by_times: Error! ',E) + print('get_INFRATEC_filter_by_times: Error! ', E) goon=False if goon: - return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']['filter'] + return True, signal_list['dimensions'], signal_list['values'][0]['meta-data']['filter'] else: return False, 0,-1 -def get_exposure_by_program(port,program,version=0,testmode=False,verbose=0): +def get_exposure_by_program(port, program, version=0, testmode=False, verbose=0): """ """ # prog=get_program_from_PID(program) @@ -1396,26 +1430,26 @@ def get_exposure_by_program(port,program,version=0,testmode=False,verbose=0): # stoptime=prog[1]['trigger']['6'][0] exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if exist: - return get_exposure_by_times(port,starttime,stoptime,version,testmode=testmode) + return get_exposure_by_times(port, starttime, stoptime, version, testmode=testmode) else: # except Exception as E: print("get_exposure_by_program: Error! cannot find the program, no exposure time available;")#, E) - return False,0,0 + return False, 0, 0 -def get_exposure_by_times(port,starttime,stoptime,version=0,testmode=False): +def get_exposure_by_times(port, starttime, stoptime, version=0, testmode=False): """ """ - OP=IR_tools.get_OP_by_time(time_ns=stoptime) + OP = IR_tools.get_OP_by_time(time_ns=stoptime) if testmode: larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_" else: larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_" stream=portpathdict[OP]["AEF"+str(port)]+"meta_"+"DATASTREAM" - if version==0: - version=get_latest_version(stream,t_from=starttime, testmode=testmode) + if version == 0: + version = get_latest_version(stream, t_from=starttime, testmode=testmode) return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/3/exposuretime"+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) -def get_camera_temp_by_program(port,program,version=0,testmode=False,verbose=0): +def get_camera_temp_by_program(port, program, version=0, testmode=False, verbose=0): """ """ ## prog=AKF_1.get_program_from_PID(program) @@ -1427,29 +1461,29 @@ def get_camera_temp_by_program(port,program,version=0,testmode=False,verbose=0): # stoptime=prog[0]['trigger']['6'][0] exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if exist: - return get_camera_temp_by_times(port,starttime,stoptime,version,testmode=testmode) + return get_camera_temp_by_times(port, starttime, stoptime, version, testmode=testmode) else: #except: print("get_camera_temp_by_program: Error! cannot find the program") - return False,0,0 + return False, 0, 0 -def get_camera_temp_by_times(port,starttime,stoptime,version=0,testmode=False): +def get_camera_temp_by_times(port, starttime, stoptime, version=0, testmode=False): """ """ - OP=IR_tools.get_OP_by_time(time_ns=stoptime) + OP = IR_tools.get_OP_by_time(time_ns=stoptime) stream=portpathdict[OP]["AEF"+str(port)]+"meta_" if testmode: larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_" else: larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_" - if version==0: - version=get_latest_version(stream+"DATASTREAM",t_from=starttime,testmode=testmode) + if version == 0: + version = get_latest_version(stream+"DATASTREAM", t_from=starttime, testmode=testmode) dummy=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) if dummy[0]: return dummy else: return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratempetarure/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) -def get_sensor_temp_by_program(port,program,version=0,testmode=False,verbose=0): +def get_sensor_temp_by_program(port, program, version=0, testmode=False, verbose=0): """ """ # prog=get_program_from_PID(program) @@ -1461,25 +1495,25 @@ def get_sensor_temp_by_program(port,program,version=0,testmode=False,verbose=0): # stoptime=prog[1]['trigger']['6'][0] exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if exist: - return get_sensor_temp_by_times(port,starttime,stoptime,version,testmode=testmode) + return get_sensor_temp_by_times(port, starttime, stoptime, version, testmode=testmode) else:#except: print("get_sensor_temp_by_program: Error! cannot find the program") - return False,0,0 + return False, 0, 0 -def get_sensor_temp_by_times(port,starttime,stoptime,version=0,testmode=False): +def get_sensor_temp_by_times(port, starttime, stoptime, version=0, testmode=False): """ """ - OP=IR_tools.get_OP_by_time(time_ns=stoptime) + OP = IR_tools.get_OP_by_time(time_ns=stoptime) stream=portpathdict[OP]["AEF"+str(port)]+"meta_" if testmode: larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_" else: larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_" - if version==0: - version=get_latest_version(stream+"DATASTREAM",t_from=starttime,testmode=testmode) + if version == 0: + version = get_latest_version(stream+"DATASTREAM", t_from=starttime, testmode=testmode) return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) -def get_camera_framerate_by_program(port,program,version=0,testmode=False,verbose=0): +def get_camera_framerate_by_program(port, program, version=0, testmode=False, verbose=0): """ """ # prog=get_program_from_PID(program) @@ -1491,27 +1525,27 @@ def get_camera_framerate_by_program(port,program,version=0,testmode=False,verbos # stoptime=prog[1]['trigger']['6'][0] exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if exist: - return get_camera_framerate_by_times(port,starttime,stoptime,version,testmode=testmode) + return get_camera_framerate_by_times(port, starttime, stoptime, version, testmode=testmode) # except: else: print("get_camera_framerate_by_program: Error! cannot find the program") - return False,0,0 + return False, 0, 0 -def get_camera_framerate_by_times(port,starttime,stoptime,version=0,testmode=False): +def get_camera_framerate_by_times(port, starttime, stoptime, version=0, testmode=False): """ """ - OP=IR_tools.get_OP_by_time(time_ns=stoptime) + OP = IR_tools.get_OP_by_time(time_ns=stoptime) stream=portpathdict[OP]["AEF"+str(port)]+"meta_" if testmode: larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_" else: larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_" - if version==0: - version=get_latest_version(stream+"DATASTREAM",t_from=starttime,testmode=testmode) + if version == 0: + version = get_latest_version(stream+"DATASTREAM", t_from=starttime, testmode=testmode) return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) -def get_frametype_by_program(port,program,version=0,testmode=False,verbose=0): +def get_frametype_by_program(port, program, version=0, testmode=False, verbose=0): """ """ ## prog=AKF_1.get_program_from_PID(program) @@ -1523,25 +1557,25 @@ def get_frametype_by_program(port,program,version=0,testmode=False,verbose=0): # stoptime=prog[0]['trigger']['6'][0] exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if exist: - return get_frametype_by_times(port,starttime,stoptime,version,testmode=testmode) + return get_frametype_by_times(port, starttime, stoptime, version, testmode=testmode) else: #except: print("get_frametype_by_program: Error! cannot find the program") - return False,0,0 + return False, 0, 0 -def get_frametype_by_times(port,starttime,stoptime,version=0,testmode=False): +def get_frametype_by_times(port, starttime, stoptime, version=0, testmode=False): """ """ - OP=IR_tools.get_OP_by_time(time_ns=starttime) + OP = IR_tools.get_OP_by_time(time_ns=starttime) stream=portpathdict[OP]["AEF"+str(port)]+"meta_" if testmode: larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_" else: larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_" - if version==0: - version=get_latest_version(stream+"DATASTREAM",t_from=starttime,testmode=testmode) + if version == 0: + version = get_latest_version(stream+"DATASTREAM", t_from=starttime, testmode=testmode) return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/5/frametype/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) -def get_framenumber_by_program(port,program,version=0,testmode=False,verbose=0): +def get_framenumber_by_program(port, program, version=0, testmode=False, verbose=0): """ """ # prog=AKF_1.get_program_from_PID(program) @@ -1553,25 +1587,25 @@ def get_framenumber_by_program(port,program,version=0,testmode=False,verbose=0): # stoptime=prog[0]['trigger']['6'][0] exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if exist: - return get_framenumber_by_times(port,starttime,stoptime,version,testmode=testmode) + return get_framenumber_by_times(port, starttime, stoptime, version, testmode=testmode) else: #except: print("get_framenumber_by_program: Error! cannot find the program") - return False,0,0 + return False, 0, 0 -def get_framenumber_by_times(port,starttime,stoptime,version=0,testmode=False): +def get_framenumber_by_times(port, starttime, stoptime, version=0, testmode=False): """ """ - OP=IR_tools.get_OP_by_time(time_ns=starttime) + OP = IR_tools.get_OP_by_time(time_ns=starttime) stream=portpathdict[OP]["AEF"+str(port)]+"meta_" if testmode: larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_" else: larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"meta_" - if version==0: - version=get_latest_version(stream+"DATASTREAM",t_from=starttime,testmode=testmode) + if version == 0: + version = get_latest_version(stream+"DATASTREAM", t_from=starttime, testmode=testmode) return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/4/framenumber/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) -def get_average_divertor_TC_for_camtemp(port,starttime,stoptime,part="all",TC=0): +def get_average_divertor_TC_for_camtemp(port, starttime, stoptime, part="all", TC=0): """ """ urlstart="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.115/" @@ -1605,22 +1639,22 @@ def get_average_divertor_TC_for_camtemp(port,starttime,stoptime,part="all",TC=0) if part=="all": rs=0 re=10 - elif part in ["h","horizontal","full h","full horizontal"]: + elif part in ["h", "horizontal", "full h", "full horizontal"]: rs=0 re=7 - elif part in ["v","vertical","vertikal"]: + elif part in ["v", "vertical", "vertikal"]: rs=7 re=10 - elif part in ["hli","horizontal low iota"]: + elif part in ["hli", "horizontal low iota"]: rs=0#4 re=4#7 - elif part in ["hhi","horizontal high iota","high iota","hi"]: + elif part in ["hhi", "horizontal high iota", "high iota", "hi"]: rs=4#0 re=7#4 elif part in [0,1,2,3,4,5,6,7,8,9,10]: rs=part re=part+1 - elif part in ["Single","s","single"]: + elif part in ["Single", "s", "single"]: rs=TC re=TC+1 else: @@ -1642,18 +1676,18 @@ def get_average_divertor_TC_for_camtemp(port,starttime,stoptime,part="all",TC=0) time=0 success=False except Exception as E: - print('get_average_divertor_TC_for_camtemp: Error! ',E) + print('get_average_divertor_TC_for_camtemp: Error! ', E) success=False try: - return success,time,signal/n + return success, time, signal/n except Exception as E: - print('get_average_divertor_TC_for_camtemp: Error! ',E) - return False,0,-1 + print('get_average_divertor_TC_for_camtemp: Error! ', E) + return False, 0,-1 else: print("get_average_divertor_TC_for_camtemp: Error! rs and re are the same numbers") - return False,0,-1 + return False, 0,-1 -def get_divertor_TC_coordinate(cameraport,TC=0): +def get_divertor_TC_coordinate(cameraport, TC=0): """ """ from IR_config_constants import TC_database @@ -1665,7 +1699,7 @@ def get_divertor_TC_coordinate(cameraport,TC=0): raise Exception("get_divertor_TC_coordinate: unknown port!") return TC_database[portnr][TC] -def get_TC_data_for_port_by_program(port,program,part=0,version=0,verbose=0): +def get_TC_data_for_port_by_program(port, program, part=0, version=0, verbose=0): """ """ ## prog=AKF_1.get_program_from_PID(program) @@ -1677,12 +1711,12 @@ def get_TC_data_for_port_by_program(port,program,part=0,version=0,verbose=0): # stoptime=prog[0]['trigger']['6'][0] exist, _, starttime, stoptime = get_trigger_from_PID(program, port, testmode=False, verbose=verbose-1) if exist: - return get_TC_data_for_port(port,starttime,stoptime,part,version) + return get_TC_data_for_port(port, starttime, stoptime, part, version) else: #except: print("get_TC_data_for_port_by_program: Error! cannot find the program") - return False,0,0 + return False, 0, 0 -def get_TC_data_for_port(port,starttime,stoptime,part=0,version=0): +def get_TC_data_for_port(port, starttime, stoptime, part=0, version=0): """ """ urlstart="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.115/DataModuleDesc.19471_DATASTREAM/" @@ -1706,7 +1740,7 @@ def get_TC_data_for_port(port,starttime,stoptime,part=0,version=0): else: success=False except Exception as E: - print('get_TC_data_for_port: Error! ',E) + print('get_TC_data_for_port: Error! ', E) success=False n=1 else: @@ -1722,33 +1756,33 @@ def get_TC_data_for_port(port,starttime,stoptime,part=0,version=0): success=False n=1 except Exception as E: - print('get_TC_data_for_port: Error! ',E) + print('get_TC_data_for_port: Error! ', E) success=False n=1 except Exception as E: - print('get_TC_data_for_port: Error! ',E) + print('get_TC_data_for_port: Error! ', E) success=False n=1 except: success=False n=1 - return success,time,signal/n + return success, time, signal/n #%% end of download functions, below combination of tools to get temperature -#def apply_LUT_to_images(LUT,images): +#def apply_LUT_to_images(LUT, images): # LUTd={LUT[0][i]: LUT[1][i] for i in range(0, len(LUT[0]))} # return np.vectorize(LUTd.get)(images) -def temp_from_LUT(LUT,images): - return LUT[0,images] +def temp_from_LUT(LUT, images): + return LUT[0, images] -def terror_from_LUT(LUT,images): - return LUT[1,images] +def terror_from_LUT(LUT, images): + return LUT[1, images] -def check_temperature_range(time,maxvalue_C,minvalue_C,port,exposure,cfilter=0): +def check_temperature_range(time,maxvalue_C,minvalue_C, port, exposure, cfilter=0): """ """ try: @@ -1772,8 +1806,8 @@ def check_temperature_range(time,maxvalue_C,minvalue_C,port,exposure,cfilter=0): raise Exception('check_temperature_range: Error! '+E) -def get_temp_from_raw_by_program_V1(portnr,program,time_window=0,emi=0.8,divertorpart="all", - version=0,intervalSize=1E9,give_ERROR=False,testmode=False,verbose=0): +def get_temp_from_raw_by_program_V1(portnr, program, time_window=0, emi=0.8,divertorpart="all", + version=0, intervalSize=1E9, give_ERROR=False, testmode=False, verbose=0): """ """ @@ -1788,9 +1822,9 @@ def get_temp_from_raw_by_program_V1(portnr,program,time_window=0,emi=0.8,diverto exist, t0, t1, t6 = get_trigger_from_PID(program, portnr, testmode, verbose=verbose-1) if not exist: print("get_temp_from_raw_by_program_V1: Error! cannot find program") - return False,0,-1,False + return False, 0,-1, False - background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0=t0,t1=t1,t6=t6,emissivity=emi,T_version=1,version=version,testmode=testmode,verbose=verbose-1) + background, LUT,refT, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error=get_calib_data(portnr, t0=t0, t1=t1, t6=t6, emissivity=emi, T_version=1, version=version, testmode=testmode, verbose=verbose-1) starttime=t1-10 if type(time_window)==list: tstart=time_window[0] @@ -1805,11 +1839,11 @@ def get_temp_from_raw_by_program_V1(portnr,program,time_window=0,emi=0.8,diverto else: stoptime=int(starttime+time_window*1e9) success=True - if verbose>0: - print(datetime.datetime.now(),"get_temp_from_raw_by_program_V1: Start download of raw images") + if verbose > 0: + print(datetime.datetime.now(), "get_temp_from_raw_by_program_V1: Start download of raw images") if (stoptime-starttime)/intervalSize>1: nrinterv=int(np.ceil((stoptime-starttime)/intervalSize)) - if verbose>0: + if verbose > 0: print("get_temp_from_raw_by_program_V1: time_window too large, splitting into {0} smaller fractions".format(nrinterv)) intervalls=[] for i in range(nrinterv): @@ -1818,22 +1852,22 @@ def get_temp_from_raw_by_program_V1(portnr,program,time_window=0,emi=0.8,diverto temperatureimages=[] times=[] for i in range(nrinterv): - raw_dl=download_raw_images_by_times(portnr,intervalls[i],intervalls[i+1],version,intervalSize,testmode=testmode,verbose=verbose-1) - print(datetime.datetime.now(),"get_temp_from_raw_by_program_V1: download of raw images part {0} of {1} finished".format(i+1, nrinterv)) + raw_dl=download_raw_images_by_times(portnr, intervalls[i], intervalls[i+1], version, intervalSize, testmode=testmode, verbose=verbose-1) + print(datetime.datetime.now(), "get_temp_from_raw_by_program_V1: download of raw images part {0} of {1} finished".format(i+1, nrinterv)) if raw_dl[0]: time=raw_dl[1] images=raw_dl[2] del raw_dl # images=images.swapaxes(1,2) images=[im.swapaxes(0,1) for im in images] - sucess,images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,True,verbose=verbose-1) + success, images=apply_calib_on_raw(images, background, LUT,refT, gain, offset, True, verbose=verbose-1) temperatureimages=temperatureimages+images times=times+time - del time,images + del time, images else: raise Exception("get_temp_from_raw_by_program_V1: cannot download the raw images") else: - raw_dl=download_raw_images_by_times(portnr,starttime,stoptime,version,intervalSize,testmode=testmode,verbose=verbose-1) + raw_dl=download_raw_images_by_times(portnr, starttime, stoptime, version, intervalSize, testmode=testmode, verbose=verbose-1) if raw_dl[0]: times=raw_dl[1] temperatureimages=raw_dl[2] @@ -1841,40 +1875,40 @@ def get_temp_from_raw_by_program_V1(portnr,program,time_window=0,emi=0.8,diverto # temperatureimages=temperatureimages.swapaxes(1,2) temperatureimages=[im.swapaxes(0,1) for im in temperatureimages] if give_ERROR: - success,temperatureimages,error_images=apply_calib_on_raw(temperatureimages,background,LUT,refT,gain,offset,gain_error,offset_error,True,give_ERROR,verbose=verbose-1) + success, temperatureimages, error_images=apply_calib_on_raw(temperatureimages, background, LUT,refT, gain, offset, gain_error, offset_error, True, give_ERROR, verbose=verbose-1) else: - success,temperatureimages=apply_calib_on_raw(temperatureimages,background,LUT,refT,gain,offset,gain_error,offset_error,True,give_ERROR,verbose=verbose-1) + success, temperatureimages=apply_calib_on_raw(temperatureimages, background, LUT,refT, gain, offset, gain_error, offset_error, True, give_ERROR, verbose=verbose-1) else: raise Exception("get_temp_from_raw_by_program_V1: cannot download the raw images") valid=True for i in range(len(temperatureimages)): try: - check_temperature_range(times[0],np.max(temperatureimages)-273.15,np.min(temperatureimages)-273.15,portnr,t_exp,cfilter) + check_temperature_range(times[0], np.max(temperatureimages)-273.15, np.min(temperatureimages)-273.15, portnr, t_exp, cfilter) except Exception as E: - print("get_temp_from_raw_by_program_V1: Error! ",E) + print("get_temp_from_raw_by_program_V1: Error! ", E) valid=False i=len(temperatureimages) if give_ERROR: - return success,times,temperatureimages,valid,error_images + return success, times, temperatureimages, valid, error_images else: - return success,times,temperatureimages,valid + return success, times, temperatureimages, valid -def get_temp_from_raw_by_program_V2(portnr,program,time_window=0,emi=0.8,version=0, - threads=1,give_ERROR=False,use_firstframe_as_background=False, +def get_temp_from_raw_by_program_V2(portnr, program, time_window=0, emi=0.8, version=0, + threads=1, give_ERROR=False,use_firstframe_as_background=False, verbose=0): """ """ - return get_temp_from_raw_by_program(portnr,program,time_window=time_window, - emi=emi,T_version=2,version=version, - threads=threads,give_ERROR=give_ERROR, + return get_temp_from_raw_by_program(portnr, program, time_window=time_window, + emi=emi, T_version=2, version=version, + threads=threads, give_ERROR=give_ERROR, use_firstframe_as_background=use_firstframe_as_background, verbose=verbose-1) -def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=2, - version=0,threads=1,give_ERROR=False, - use_firstframe_as_background=False,back_emissivity=0.8, - verbose=0,testmode=False,framerate='max'): +def get_temp_from_raw_by_program(portnr, program, time_window=0, emi=0.8, T_version=2, + version=0, threads=1, give_ERROR=False, + use_firstframe_as_background=False, back_emissivity=0.8, + verbose=0, testmode=False, framerate='max'): """ """ FLIR=False @@ -1883,7 +1917,7 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version= raise Exception("get_temp_from_raw_by_program: unknown Program") success=True - OP=IR_tools.get_OP_by_time(time_ns=t1) + OP = IR_tools.get_OP_by_time(time_ns=t1) if type(portnr) is str: try: portnr=int(portnr) @@ -1908,10 +1942,10 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version= # --- Glens FLIR camera --- # # get raw data - exist,time,frames,mode,tsets,versions=download_raw_FLIR_images_via_png(t1=t1,t6=t6,time_window=time_window,version=version,threads=threads,verbose=verbose-1,testmode=testmode) + exist, time, frames,mode, tsets, versions=download_raw_FLIR_images_via_png(t1=t1, t6=t6, time_window=time_window, version=version, threads=threads, verbose=verbose-1, testmode=testmode) if not exist: print("get_temp_from_raw_by_program: Error! FLIR data not found") - return False,[0],[0],[] + return False, [0], [0], [] # get calibration if mode==1: Psets=IR_tools.check_dublicates(tsets) @@ -1935,7 +1969,7 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version= break except: ipi=-1 - c1=[float(pari['Coeff0_'+str(ipi)].split("\n")[0]),float(pari['Coeff1_'+str(ipi)].split("\n")[0])] + c1=[float(pari['Coeff0_'+str(ipi)].split("\n")[0]), float(pari['Coeff1_'+str(ipi)].split("\n")[0])] c2=[float(pari['TempCoeff'+str(x)+'_'+str(ipi)].split("\n")[0]) for x in range(7)] Radi_Co.append(c1) Temp_Co.append(c2) @@ -1952,7 +1986,7 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version= parlog=read_restdb_old(archivepath+"W7X/QSR07_FLIR/AEK51_raw_PS"+str(PS)+"_PARLOG/V"+str(versions[PS])+"/_signal.json?from="+str(t1-10)+"&upto="+str(t6+20)) if parlog[0]: pari=parlog[2][0]['meta-data'] - c1=[float(pari['Coeff0_'+str(PS)].split("\n")[0]),float(pari['Coeff1_'+str(PS)].split("\n")[0])] + c1=[float(pari['Coeff0_'+str(PS)].split("\n")[0]), float(pari['Coeff1_'+str(PS)].split("\n")[0])] c2=[float(pari['TempCoeff'+str(x)+'_'+str(PS)].split("\n")[0]) for x in range(7)] Radi_Co.append(c1) Temp_Co.append(c2) @@ -1963,22 +1997,22 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version= Radi_Co.append([]) Temp_Co.append([]) # got the raw, knowing which frame needs which calibration, got the calibration, now we have to transform it - if verbose>0: - print(datetime.datetime.now(),"get_temp_from_raw_by_program: Converting the raw data into temperature, number of frames:",len(frames)) + if verbose > 0: + print(datetime.datetime.now(), "get_temp_from_raw_by_program: Converting the raw data into temperature, number of frames:", len(frames)) for i in range(len(frames)): try: frames[i]=Radi_Co[tsets[i]][0]+frames[i]*Radi_Co[tsets[i]][1] dummy=frames[i] temp=Temp_Co[tsets[i]][0]+Temp_Co[tsets[i]][1]*dummy for expo in range(2,7): - temp=temp+Temp_Co[tsets[i]][expo]*np.power(dummy,expo) + temp=temp+Temp_Co[tsets[i]][expo]*np.power(dummy, expo) frames[i]=temp+273.15 except Exception as E: - print('get_temp_from_raw_by_program: Error in frame {0}! {1}'.format(i,E)) - print(' (len(tsets) {0},Radi_Co[tsets[i]] {1},Temp_Co[tsets[i]] {2})'.format(len(tsets),Radi_Co[tsets[i]],Temp_Co[tsets[i]])) - if verbose>0: - print(datetime.datetime.now(),"get_temp_from_raw_by_program: temperature frames done") - return exist,time,frames,tsets + print('get_temp_from_raw_by_program: Error in frame {0}! {1}'.format(i, E)) + print(' (len(tsets) {0},Radi_Co[tsets[i]] {1}, Temp_Co[tsets[i]] {2})'.format(len(tsets),Radi_Co[tsets[i]], Temp_Co[tsets[i]])) + if verbose > 0: + print(datetime.datetime.now(), "get_temp_from_raw_by_program: temperature frames done") + return exist, time, frames, tsets elif mode==2: raise Exception("get_temp_from_raw_by_program: not implemented in mode 2") else: @@ -1988,95 +2022,95 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version= # --- IRcam case --- # # get calibration data - cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0=t0,t1=t1,t6=t6,emissivity=emi,T_version=T_version,version=version,back_emissivity=back_emissivity,testmode=testmode,verbose=verbose-1) + cexist, background, LUT,refT, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error = get_calib_data(portnr, t0=t0, t1=t1, t6=t6, emissivity=emi, T_version=T_version, version=version, back_emissivity=back_emissivity, testmode=testmode, verbose=verbose-1) if not cexist: - if verbose>0: - print("get_temp_from_raw_by_program: Unable to load the complete calibration data for",program," please confirm that this data was uploaded") - return False,[0],[0],False + if verbose > 0: + print("get_temp_from_raw_by_program: Unable to load the complete calibration data for", program, " please confirm that this data was uploaded") + return False, [0], [0], False # get raw data from latest or requested version - if version==0: - version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM",testmode=testmode,program=program) + if version == 0: + version = get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM", testmode=testmode, program=program) if testmode: larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" else: larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" - if verbose>0: - print(datetime.datetime.now(),"get_temp_from_raw_by_program: raw download start") + if verbose > 0: + print(datetime.datetime.now(), "get_temp_from_raw_by_program: raw download start") if fastDL: - exist,time,frames=download_images_by_time_via_png(larchivepath,starttime=t1,stoptime=t6,time_window=time_window,version=version,threads=threads,verbose=verbose-1,framerate=framerate) + exist, time, frames=download_images_by_time_via_png(larchivepath, starttime=t1, stoptime=t6, time_window=time_window, version=version, threads=threads, verbose=verbose-1, framerate=framerate) else: - exist,time,frames=download_raw_images_by_program(portnr,program,time_window,version,testmode=testmode,verbose=verbose-1) + exist, time, frames=download_raw_images_by_program(portnr, program, time_window, version, testmode=testmode, verbose=verbose-1) if exist: frames=[im.swapaxes(0,1) for im in frames] if not exist: raise Warning("get_temp_from_raw_by_program: data not found in database!") - return False,[0],[0],False - if verbose>0: - print(datetime.datetime.now(),"get_temp_from_raw_by_program: raw download finished") + return False, [0], [0], False + if verbose > 0: + print(datetime.datetime.now(), "get_temp_from_raw_by_program: raw download finished") # calibrate frames FOV=get_FOV_mask(portnr) if use_firstframe_as_background: - bim=apply_NUC([frames[0]],gain,offset) - background=get_average_background_recangle(portnr,bim[0]) + bim=apply_NUC([frames[0]], gain, offset) + background=get_average_background_recangle(portnr, bim[0]) if give_ERROR: - success,frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1) + success, frames, error_images=apply_calib_on_raw(frames, background, LUT,refT, gain, offset, gain_error, offset_error, False, give_ERROR, verbose=verbose-1) else: - success,frames=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1) - if verbose>0: - print(datetime.datetime.now(),"get_temp_from_raw_by_program: Temperature calculation done") + success, frames=apply_calib_on_raw(frames, background, LUT,refT, gain, offset, gain_error, offset_error, False, give_ERROR, verbose=verbose-1) + if verbose > 0: + print(datetime.datetime.now(), "get_temp_from_raw_by_program: Temperature calculation done") if not success: - if verbose>0: + if verbose > 0: print("calculation of the temperature failed, calibration could not be applied") - return False,time,frames,False + return False, time, frames, False # crop to FOV for i in range(len(frames)): frames[i]=(frames[i]*FOV).astype(np.float32) # correct bad pixels - if verbose>0: - print(datetime.datetime.now(),"get_temp_from_raw_by_program: correcting bad pixels") - frames=IR_tools.correct_images(frames,badpixels) + if verbose > 0: + print(datetime.datetime.now(), "get_temp_from_raw_by_program: correcting bad pixels") + frames = IR_tools.correct_images(frames, badpixels) # TODO: check temperature range for saturation and mark frames as not valid # -# if verbose>0: -# print(datetime.datetime.now(),"get_temp_from_raw_by_program: checking temperaturerange") - valid=True +# if verbose > 0: +# print(datetime.datetime.now(), "get_temp_from_raw_by_program: checking temperaturerange") + valid = True # for i in range(len(frames)): # try: -# check_temperature_range(time[0],np.max(frames)-273.15,np.min(frames[:][np.nonzero(frames[i])])-273.15,portnr,t_exp,cfilter) +# check_temperature_range(time[0], np.max(frames)-273.15, np.min(frames[:][np.nonzero(frames[i])])-273.15, portnr, t_exp, cfilter) # except Exception as E: # print(E) # valid=False # i=len(frames) # check for time vector offsets - date=int(program.split(".")[0]) - if date>20180712 and date<20180720: - toff=time[0]-t1 - if toff>10e6: + date = int(program.split(".")[0]) + if date > 20180712 and date < 20180720: + toff = time[0]-t1 + if toff > 10e6: print("get_temp_from_raw_by_program: time offset of {0}ns detected, correcting".format(toff)) - time[:]=time[:]-toff + time[:] = time[:]-toff if give_ERROR: - return exist,time,frames,valid,error_images + return exist, time, frames, valid, error_images else: - return exist,time,frames,valid + return exist, time, frames, valid -def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8, - T_version=2,version=0,threads=1, - give_ERROR=False,check_range=True, - testmode=False,verbose=0): +def get_temp_from_raw_by_program_fullthreads(portnr, program, time_window=0, emi=0.8, + T_version=2, version=0, threads=1, + give_ERROR=False, check_range=True, + testmode=False, verbose=0): """ """ - if threads==1 or not fastDL: - return get_temp_from_raw_by_program(portnr,program,time_window,emi, - T_version,version,threads,give_ERROR, + if threads == 1 or not fastDL: + return get_temp_from_raw_by_program(portnr, program, time_window, emi, + T_version, version, threads, give_ERROR, testmode=testmode, verbose=verbose-1) else: if type(portnr) is str: try: - OP=IR_tools.get_OP_by_time(program_str=program) - camera=portcamdict[OP][portnr] + OP = IR_tools.get_OP_by_time(program_str=program) + camera = portcamdict[OP][portnr] if camera.split("_")[0]=='FLIR': FLIR=True else: @@ -2086,11 +2120,11 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0. except Exception as E: raise Exception("get_temp_from_raw_by_program_fullthreads: unknown Port!"+E) elif type(portnr) is int: - FLIR=False - goon=True - portnr=int(portnr) + FLIR = False + goon = True + portnr = int(portnr) else: - goon=False + goon = False if goon and not FLIR: # prog=get_program_from_PID(program) # if prog[0]: @@ -2098,60 +2132,60 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0. # stoptime=prog[1]['trigger']['6'][0] exist, t0, starttime, stoptime = get_trigger_from_PID(program, portnr, testmode, verbose=verbose-1) if exist: - exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0=t0,t1=starttime,t6=stoptime,emissivity=emi,T_version=T_version,version=version,testmode=testmode,verbose=verbose-1) - FOV=get_FOV_mask(portnr) - success=True - OP=IR_tools.get_OP_by_time(time_ns=starttime) - if version==0: - version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM",program=program,testmode=testmode) + exist, background, LUT,refT, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error = get_calib_data(portnr, t0=t0, t1=starttime, t6=stoptime, emissivity=emi, T_version=T_version, version=version, testmode=testmode, verbose=verbose-1) + FOV = get_FOV_mask(portnr) + success = True + OP = IR_tools.get_OP_by_time(time_ns=starttime) + if version == 0: + version = get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM", program=program, testmode=testmode) if testmode: - larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" else: - larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" - if type(time_window)==list: - tstart=time_window[0] - tstop=time_window[1] - if tstop<tstart: + larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + if type(time_window) == list: + tstart = time_window[0] + tstop = time_window[1] + if tstop < tstart: raise Exception("get_temp_from_raw_by_program_fullthreads: t_stop before t_start") - stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart) - stdate=stdate.isoformat() - enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop) - enddate=enddate.isoformat() + stdate = datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart) + stdate = stdate.isoformat() + enddate = datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop) + enddate = enddate.isoformat() else: - stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9) - stdate=stdate.isoformat() - if time_window==0: - enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9) - enddate=enddate.isoformat() + stdate = datetime.datetime.utcfromtimestamp((starttime-100)/1e9) + stdate = stdate.isoformat() + if time_window == 0: + enddate = datetime.datetime.utcfromtimestamp(stoptime/1e9) + enddate = enddate.isoformat() else: - enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window) - enddate=enddate.isoformat() + enddate = datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window) + enddate = enddate.isoformat() - times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))# - lnt=len(times) - tim=[] + times = AKF_2.get_time_intervals(larchivepath, stdate.replace("T", " "), enddate.replace("T", " "))# + lnt = len(times) + tim = [] for i in range(lnt): tim.append(times[lnt-1-i][0]) - intervalls=[] - intervalSize=int(lnt/threads) + intervalls = [] + intervalSize = int(lnt/threads) for i in range(threads): intervalls.append(int(i*intervalSize)) intervalls.append(lnt) jobs = [] resultdict = [] for i in range(threads): - if verbose>0: - print("get_temp_from_raw_by_program_fullthreads: Start Thread ",i+1) - # p = multiprocessing.Process(target=convert_raw_to_temp_thread, args=(larchivepath,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,)) - p=convert_raw_to_temp_thread(larchivepath,tim[intervalls[i]:intervalls[i+1]], - resultdict,i,version,background,LUT, - refT,gain,offset,gain_error,offset_error, - give_ERROR,FOV,badpixels,verbose=verbose-1) + if verbose > 0: + print("get_temp_from_raw_by_program_fullthreads: Start Thread ", i+1) + # p = multiprocessing.Process(target=convert_raw_to_temp_thread, args=(larchivepath, tim[intervalls[i]:intervalls[i+1]], out_q, i, version, background, LUT,refT, gain, offset, gain_error, offset_error, give_ERROR, fOV, badpixels,)) + p=convert_raw_to_temp_thread(larchivepath, tim[intervalls[i]:intervalls[i+1]], + resultdict, i, version, background, LUT, + refT, gain, offset, gain_error, offset_error, + give_ERROR, FOV, badpixels, verbose=verbose-1) jobs.append(p) p.start() for p in jobs: p.join() - if verbose>0: + if verbose > 0: print("get_temp_from_raw_by_program_fullthreads: all threads are done") order=[] for ele in resultdict: @@ -2165,14 +2199,14 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0. times=times+resultdict[order.index(i)][2] del resultdict if check_range: - if verbose>0: - print(datetime.datetime.now(),"get_temp_from_raw_by_program_fullthreads: checking temperaturerange") + if verbose > 0: + print(datetime.datetime.now(), "get_temp_from_raw_by_program_fullthreads: checking temperaturerange") valid=True for i in range(len(images)): try: - check_temperature_range(times[0],np.max(images[i])-273.15,np.min(images[i][np.nonzero(images[i])])-273.15,portnr,t_exp,cfilter) + check_temperature_range(times[0], np.max(images[i])-273.15, np.min(images[i][np.nonzero(images[i])])-273.15, portnr, t_exp, cfilter) except Exception as E: - print('get_temp_from_raw_by_program_fullthreads: Error! ',E) + print('get_temp_from_raw_by_program_fullthreads: Error! ', E) valid=False i=len(images) else: @@ -2185,54 +2219,54 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0. error_images=[0] if give_ERROR: - return success,np.array(times),images,valid,error_images + return success, np.array(times), images, valid, error_images else: - return success,np.array(times),images,valid + return success, np.array(times), images, valid -def get_nuced_raw_by_program(portnr,program,time_window=0,version=0, - emi=1,T_version=2,threads=1, - give_ERROR=False,testmode=False,verbose=0): +def get_nuced_raw_by_program(portnr, program, time_window=0,version=0, + emi=1, T_version=2, threads=1, + give_ERROR=False, testmode=False, verbose=0): """ """ # get calibration data - exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program=program,emissivity=emi,T_version=T_version,version=version,testmode=testmode,verbose=verbose-1) - if verbose>0: - print(datetime.datetime.now(),"get_nuced_raw_by_program: raw download start") + exist, background, LUT,refT, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error=get_calib_data(portnr, program=program, emissivity=emi, T_version=T_version, version=version, testmode=testmode, verbose=verbose-1) + if verbose > 0: + print(datetime.datetime.now(), "get_nuced_raw_by_program: raw download start") if fastDL: - exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_window,version, - threads,testmode=testmode,verbose=verbose-1) + exist, time, frames=download_raw_images_by_program_via_png(portnr, program, time_window, version, + threads, testmode=testmode, verbose=verbose-1) else: - exist,time,frames=download_raw_images_by_program(portnr,program,time_window,version,testmode=testmode,verbose=verbose-1) + exist, time, frames=download_raw_images_by_program(portnr, program, time_window, version, testmode=testmode, verbose=verbose-1) if exist: frames=[im.swapaxes(0,1) for im in frames] else: raise Exception("get_nuced_raw_by_program: no data found") - if verbose>0: - print(datetime.datetime.now(),"get_nuced_raw_by_program: raw download finished") + if verbose > 0: + print(datetime.datetime.now(), "get_nuced_raw_by_program: raw download finished") FOV=get_FOV_mask(portnr) if give_ERROR: - frames,frames_error=apply_NUC(frames,gain,offset,gain_error,offset_error,give_ERROR) + frames, frames_error=apply_NUC(frames, gain, offset, gain_error, offset_error, give_ERROR) else: - frames=apply_NUC(frames,gain,offset,gain_error,offset_error,give_ERROR) + frames=apply_NUC(frames, gain, offset, gain_error, offset_error, give_ERROR) for i in range(len(frames)): frames[i]=(frames[i]*FOV).astype(np.float32) - if verbose>0: - print(datetime.datetime.now(),"get_nuced_raw_by_program: correcting bad pixels") - frames=IR_tools.correct_images(frames,badpixels,verbose=verbose-1) - if verbose>0: - print(datetime.datetime.now(),"get_nuced_raw_by_program: checking temperaturerange") + if verbose > 0: + print(datetime.datetime.now(), "get_nuced_raw_by_program: correcting bad pixels") + frames=IR_tools.correct_images(frames, badpixels, verbose=verbose-1) + if verbose > 0: + print(datetime.datetime.now(), "get_nuced_raw_by_program: checking temperaturerange") valid=True if give_ERROR: - return exist,time,frames,valid,frames_error + return exist, time, frames, valid, frames_error else: - return exist,time,frames,valid + return exist, time, frames, valid -def get_nuced_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8, - T_version=2,version=1,threads=1, - give_ERROR=False,check_range=True, - testmode=False,verbose=0): +def get_nuced_raw_by_program_fullthreads(portnr, program, time_window=0, emi=0.8, + T_version=2, version=1, threads=1, + give_ERROR=False, check_range=True, + testmode=False, verbose=0): """ """ # prog=get_program_from_PID(program) @@ -2245,48 +2279,48 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8, return False, None, None, None if threads==1 or not fastDL: - return get_nuced_raw_by_program(portnr,program, - time_window=time_window,emi=emi, - T_version=T_version,version=version, - threads=threads,give_ERROR=give_ERROR, + return get_nuced_raw_by_program(portnr, program, + time_window=time_window, emi=emi, + T_version=T_version, version=version, + threads=threads, give_ERROR=give_ERROR, testmode=testmode, verbose=verbose-1) else: - exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,t0=t0,t1=starttime,t6=stoptime,emissivity=emi,T_version=T_version,version=version,testmode=testmode,verbose=verbose-1) + exist, background, LUT,refT, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error=get_calib_data(portnr, t0=t0, t1=starttime, t6=stoptime, emissivity=emi, T_version=T_version, version=version, testmode=testmode, verbose=verbose-1) FOV=get_FOV_mask(portnr) success=True - OP=IR_tools.get_OP_by_time(time_ns=starttime) - if verbose>0: + OP = IR_tools.get_OP_by_time(time_ns=starttime) + if verbose > 0: print('get_nuced_raw_by_program_fullthreads: prepare loading images by threads') - if version==0: - version=get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM",t_from=starttime,testmode=testmode) + if version == 0: + version = get_latest_version(portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM", t_from=starttime, testmode=testmode) if testmode: larchivepath=testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" else: larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(portnr)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" - if type(time_window)==list: - tstart=time_window[0] - tstop=time_window[1] - if tstop<tstart: + if type(time_window) == list: + tstart = time_window[0] + tstop = time_window[1] + if tstop < tstart: raise Exception("get_nuced_raw_by_program_fullthreads: t_stop before t_start") - stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart) - stdate=stdate.isoformat() - enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop) - enddate=enddate.isoformat() + stdate = datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart) + stdate = stdate.isoformat() + enddate = datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop) + enddate = enddate.isoformat() else: - stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9) - stdate=stdate.isoformat() - if time_window==0: - enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9) - enddate=enddate.isoformat() + stdate = datetime.datetime.utcfromtimestamp((starttime-100)/1e9) + stdate = stdate.isoformat() + if time_window == 0: + enddate = datetime.datetime.utcfromtimestamp(stoptime/1e9) + enddate = enddate.isoformat() else: - enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window) - enddate=enddate.isoformat() - times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))# - lnt=len(times) - tim=[] + enddate = datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window) + enddate = enddate.isoformat() + times = AKF_2.get_time_intervals(larchivepath, stdate.replace("T", " "), enddate.replace("T", " "))# + lnt = len(times) + tim = [] for i in range(lnt): tim.append(times[lnt-1-i][0]) - intervalls=[] + intervalls = [] intervalSize=int(lnt/threads) for i in range(threads): intervalls.append(int(i*intervalSize)) @@ -2294,31 +2328,31 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8, jobs = [] resultdict = [] for i in range(threads): - if verbose>0: - print("get_nuced_raw_by_program_fullthreads: Start Thread ",i+1) -# p = multiprocessing.Process(target=NUC_raw_thread, args=(portnr,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,background,LUT,refT,gain,offset,gain_error,offset_error,give_ERROR,FOV,badpixels,)) - p = NUC_raw_thread(larchivepath,tim[intervalls[i]:intervalls[i+1]], - resultdict,i,version,background,LUT,refT, - gain,offset,gain_error,offset_error,give_ERROR, - FOV,badpixels,verbose=verbose-1) + if verbose > 0: + print("get_nuced_raw_by_program_fullthreads: Start Thread ", i+1) +# p = multiprocessing.Process(target=NUC_raw_thread, args=(portnr, tim[intervalls[i]:intervalls[i+1]], out_q, i, version, background, LUT,refT, gain, offset, gain_error, offset_error, give_ERROR, fOV, badpixels,)) + p = NUC_raw_thread(larchivepath, tim[intervalls[i]:intervalls[i+1]], + resultdict, i, version, background, LUT,refT, + gain, offset, gain_error, offset_error, give_ERROR, + FOV, badpixels, verbose=verbose-1) jobs.append(p) p.start() for p in jobs: p.join() - if verbose>0: + if verbose > 0: print("get_nuced_raw_by_program_fullthreads: all threads are done") - order=[] + order = [] for ele in resultdict: order.append(ele[0]) if len(np.where(np.asarray(ele[1])==False)[0])>0: - success=False - times=[]#np.array(resultdict[order.index(0)][2]) - images=[]#np.array(resultdict[order.index(0)][3]) + success = False + times = []#np.array(resultdict[order.index(0)][2]) + images = []#np.array(resultdict[order.index(0)][3]) for i in range(threads): - images=images+resultdict[order.index(i)][3] - times=times+resultdict[order.index(i)][2] + images = images+resultdict[order.index(i)][3] + times = times+resultdict[order.index(i)][2] del resultdict - valid=True + valid = True # else: # success=False # times=[0] @@ -2328,23 +2362,23 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_window=0,emi=0.8, # print(larchivepath) if give_ERROR: - return success,np.array(times),images,valid#,error_images + return success, np.array(times), images, valid#, error_images else: - return success,np.array(times),images,valid + return success, np.array(times), images, valid -def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_error=0, - offset_error=0,fullbackground=False,give_ERROR=False,verbose=0): +def apply_calib_on_raw(images, background, LUT,refT=28.5, gain=0, offset=0, gain_error=0, + offset_error=0, fullbackground=False, give_ERROR=False, verbose=0): """ apply_calib_on_raw - return success,images and if give_Error: errorimages + return success, images and if give_Error: errorimages """ try: #images=np.array(raw,dtype=np.uint16) # del raw # images=images.swapaxes(1,2) if type(gain)!=int and type(offset)!=int: - if verbose>0: - print(datetime.datetime.now(),"apply_calib_on_raw: NUCing") + if verbose > 0: + print(datetime.datetime.now(), "apply_calib_on_raw: NUCing") # eliminate bad offset and gain points offset[offset==np.inf] = 0 offset[offset==-np.inf] = 0 @@ -2352,11 +2386,11 @@ def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_erro gain[gain==-np.inf] = 1 # try to apply NUC if give_ERROR: - images,error_images=apply_NUC(images,gain,offset,gain_error,offset_error,give_ERROR) + images, error_images=apply_NUC(images, gain, offset, gain_error, offset_error, give_ERROR) else: - images=apply_NUC(images,gain,offset) - if verbose>0: - print(datetime.datetime.now(),"apply_calib_on_raw: background treatment") + images=apply_NUC(images, gain, offset) + if verbose > 0: + print(datetime.datetime.now(), "apply_calib_on_raw: background treatment") # if fullbackground: #sbackground=np.zeros(np.shape(images[0]),dtype=np.float32)+background for i in range(len(images)): @@ -2366,34 +2400,34 @@ def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_erro # images[i]=images[i]-background # images=np.array(images.clip(min=0),dtype=np.uint16) # images=[np.round(im.clip(min=0)).astype(np.uint16) for im in images] - if verbose>0: - print(datetime.datetime.now(),"apply_calib_on_raw: applying LUT") - LUT=np.array([LUT[1],LUT[2]]) + if verbose > 0: + print(datetime.datetime.now(), "apply_calib_on_raw: applying LUT") + LUT=np.array([LUT[1], LUT[2]]) if give_ERROR: terror=[]#np.zeros(np.shape(images)) for i in range(len(images)): # uncertainty of temperature calibration - terror=terror+[terror_from_LUT(LUT,images[i])] + terror=terror+[terror_from_LUT(LUT, images[i])] # error propagation of uncertainty in NUC - max_image = temp_from_LUT(LUT,images[i]+error_images[i]) - min_image = temp_from_LUT(LUT,images[i]-error_images[i]) + max_image = temp_from_LUT(LUT, images[i]+error_images[i]) + min_image = temp_from_LUT(LUT, images[i]-error_images[i]) terror[i] = (terror[i] + (max_image - min_image)/2).astype(np.float32) - images[i]=(temp_from_LUT(LUT,images[i])).astype(np.float32) + images[i]=(temp_from_LUT(LUT, images[i])).astype(np.float32) images[i]=(images[i]+(refT+273.15)).astype(np.float32) - return True,images,terror + return True, images, terror else: for i in range(len(images)): - images[i]=(temp_from_LUT(LUT,images[i])).astype(np.float32) + images[i]=(temp_from_LUT(LUT, images[i])).astype(np.float32) images[i]=(images[i]+(refT+273.15)).astype(np.float32) - return True,images + return True, images except Exception as E: raise Warning('apply_calib_on_raw: '+E) - return False,[0] +# return False, [0] -def apply_NUC(images,gain,offset,gain_error=0,offset_error=0,give_error=False): +def apply_NUC(images, gain, offset, gain_error=0, offset_error=0, give_error=False): """ - apply_NUC(images=numpy array(time,width,height),gain,offset =numpy array(width,height) ) + apply_NUC(images=numpy array(time,width,height), gain, offset =numpy array(width,height) ) """ try: if give_error: @@ -2430,7 +2464,7 @@ def get_cooling_water_temperature_by_time(time_ns): except: print("get_cooling_water_temperature_by_time: inlet water temperature not found") inlet=0 - result_in=[False,0,0] + result_in=[False, 0, 0] try: result_out=read_restdb(url_base+outlet_url+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) if result_out[0]: @@ -2441,14 +2475,14 @@ def get_cooling_water_temperature_by_time(time_ns): except: print("get_cooling_water_temperature_by_time: outlet water temperature not found") inlet=0 - result_out=[False,0,0] - return result_in[0] and result_out[0],inlet,outlet + result_out=[False, 0, 0] + return result_in[0] and result_out[0], inlet, outlet -def get_calib_data(port,program=None,t0=None,t1=None,t6=None,emissivity=0.8, - T_version=2,version=0,back_emissivity=0.82,testmode=False,verbose=0): +def get_calib_data(port, program=None, t0=None, t1=None, t6=None, emissivity=0.8, + T_version=2, version=0, back_emissivity=0.82, testmode=False, verbose=0): """ get_calib_data - return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error + return exist, background, LUT,refT, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error """ if t0 is None or t1 is None or t6 is None: if program is None: @@ -2458,58 +2492,58 @@ def get_calib_data(port,program=None,t0=None,t1=None,t6=None,emissivity=0.8, program = AKF_2.get_program_id(t1) else: raise Exception('get_calib_data: ERROR! Missing time definition!') - print(' function requires either a program ID or the trigger times t0, t1 and t6') +# print(' function requires either a program ID or the trigger times t0, t1 and t6') # get start and end times from program ID exist, t0, t1, t6 = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if not exist: raise Exception('get_calib_data: ERROR! program {0} not found!'.format(program)) # determin camera in this port and OP - OP=IR_tools.get_OP_by_time(time_ns=t0) + OP = IR_tools.get_OP_by_time(time_ns=t0) exist=True - if verbose>0: - print('get_calib_data: loading data for camera {1} in {0}'.format(OP,port)) + if verbose > 0: + print('get_calib_data: loading data for camera {1} in {0}'.format(OP, port)) Camera=portcamdict[OP]['AEF'+str(port)] - if Camera.split("_")[0]=="Infratec": -# cf=get_INFRATEC_filter_by_program(program,version) - cf=get_INFRATEC_filter_by_times(starttime=t1,stoptime=t6,testmode=testmode) + if Camera.split("_")[0] == "Infratec": +# cf=get_INFRATEC_filter_by_program(program, version) + cf = get_INFRATEC_filter_by_times(starttime=t1, stoptime=t6, testmode=testmode) if cf[0]: - cfilter=cf[2] + cfilter = cf[2] else: - cfilter=-1 + cfilter = -1 raise Warning("get_calib_data: Filter not found for INFRATEC Camera for Program "+AKF_2.get_program_id(t0)) - return False,0,0,0,0,0,0,0,0,0,0 +# return False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 else: cfilter=0 # get exposure time of first frame - expo_DL=get_exposure_by_times(port,t1,t6,version,testmode) + expo_DL=get_exposure_by_times(port, t1, t6, version, testmode) if expo_DL[0]: - t_exp=int(expo_DL[2][0]) + t_exp = int(expo_DL[2][0]) del expo_DL else: raise Warning("get_calib_data: no exposure time found") - return False,0,0,0,0,0,0,0,cfilter,0,0 +# return False, 0, 0, 0, 0, 0, 0, 0, cfilter, 0, 0 # get Look-Up-Table (LUT) - time=int(TimeToNs([2017,9,26],[8,0,0,0])) - LUT_DL=download_LUT(port,time,t_exp,emissivity,cfilter,version,testmode=testmode,verbose=verbose-1) + time = int(TimeToNs([2017,9,26], [8, 0, 0, 0])) + LUT_DL=download_LUT(port, time, t_exp, emissivity, cfilter, version, testmode=testmode, verbose=verbose-1) if LUT_DL[0]: LUT=LUT_DL[1] del LUT_DL else: raise Warning("get_calib_data: no LUT found") - return False,0,0,0,0,0,0,t_exp,cfilter,0,0 +# return False, 0, 0, 0, 0, 0, 0, t_exp, cfilter, 0, 0 # treat cameras with and w/o filters differently - if cfilter==0 and not Camera.split("_")[0]=="Infratec": + if cfilter==0 and not Camera.split("_")[0] == "Infratec": # --- Infratec Camera case --- - if verbose>0: + if verbose > 0: print('get_calib_data: getting NUC arrays') - NUC_DL=get_NUC_by_times(port,t0,t1,t_exp,version,testmode,verbose=verbose-1)#download_NUC_by_times(port,t0,t1,t_exp,version) + NUC_DL=get_NUC_by_times(port, t0, t1, t_exp, version, testmode, verbose=verbose-1)#download_NUC_by_times(port, t0, t1, t_exp, version) if NUC_DL[0]==False: raise Warning("get_calib_data: NUC was not found") - return False,0,LUT,0,0,0,0,t_exp,cfilter,0,0 +# return False, 0, LUT, 0, 0, 0, 0, t_exp, cfilter, 0, 0 # extract NUC elements gain = [] offset = [] @@ -2518,126 +2552,126 @@ def get_calib_data(port,program=None,t0=None,t1=None,t6=None,emissivity=0.8, offset_error = [] gain=np.array(NUC_DL[1][0]) offset=np.array(NUC_DL[1][1]) - badpixels=np.array(NUC_DL[1][3],dtype=np.ubyte) + badpixels=np.array(NUC_DL[1][3], dtype=np.ubyte) # check quality of NUC elements badpixels[np.isnan(gain)] = 1 badpixels[np.isnan(offset)] = 1 - gain[np.isnan(gain)]=0 - offset[np.isnan(offset)]=0 + gain[np.isnan(gain)] = 0 + offset[np.isnan(offset)] = 0 badpixels[offset<-1e100] = 1 - offset[offset<-1e100]=0 + offset[offset<-1e100] = 0 # catch cases of error frames being the value "0" if NUC_DL[1][4] is 0: gain_error = np.zeros(np.shape(gain), dtype=np.int8) else: - gain_error=np.array(NUC_DL[1][4]) + gain_error = np.array(NUC_DL[1][4]) badpixels[np.isnan(gain_error)] = 1 - gain_error[np.isnan(gain_error)]=np.nanmax(gain_error)+1 + gain_error[np.isnan(gain_error)] = np.nanmax(gain_error)+1 if NUC_DL[1][5] is 0: offset_error = np.zeros(np.shape(offset), dtype=np.int8) else: - offset_error=np.array(NUC_DL[1][5]) + offset_error = np.array(NUC_DL[1][5]) badpixels[np.isnan(offset_error)] = 1 - offset_error[np.isnan(offset_error)]=np.nanmax(offset_error)+1 + offset_error[np.isnan(offset_error)] = np.nanmax(offset_error)+1 # catch special case for AEF11 and AEF21 in OP1.2a - if ((port == 11) or (port == 21)) and OP=="OP1.2a": - exist,time,frames=download_raw_images_by_times(port,t1,int(t1+0.02*1e9),version,testmode=testmode,verbose=verbose-1) + if ((port == 11) or (port == 21)) and OP == "OP1.2a": + exist, time, frames=download_raw_images_by_times(port, t1, int(t1+0.02*1e9), version, testmode=testmode, verbose=verbose-1) if exist: # frames=[im.swapaxes(0,1) for im in frames]#somehow the archive gives it now already swapped back (09.07.2018) - bim=apply_NUC([frames[0]],gain,offset) - background=get_average_background_recangle(port,bim[0]) + bim = apply_NUC([frames[0]], gain, offset) + background = get_average_background_recangle(port, bim[0]) else: raise Warning("get_calib_data: cannot find the first frame of the discharge, reconstruction failed") - return False,0,LUT,0,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error +# return False, 0, LUT, 0, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error else: # --- Ircam Camera case --- - gain=0 - offset=0 - badpixels=0 - gain_error=0 - offset_error=0 + gain = 0 + offset = 0 + badpixels = 0 + gain_error = 0 + offset_error = 0 # get background - if OP=="OP1.2a": - back_DL=download_background_by_times(port,t0,t1,t_exp,cfilter,version,testmode=testmode) - elif OP=="OP1.2b": - back_DL=get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version,testmode,verbose=verbose-1) + if OP == "OP1.2a": + back_DL = download_background_by_times(port, t0, t1, t_exp, cfilter, version, testmode=testmode) + elif OP == "OP1.2b": + back_DL = get_NUCed_background_by_times(port, t0, t1, t_exp, cfilter, gain, offset, version, testmode, verbose=verbose-1) else: raise Exception("get_calib_data: unknown Operation Phase or calibration method not implemented for this campaign") if not back_DL[0]: raise Warning("get_calib_data: no background image found") - return False,0,LUT,0,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error - background=back_DL[2] +# return False, 0, LUT, 0, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error + background = back_DL[2] # apply different versions of temperature calibration - if T_version==1: - if verbose>0: + if T_version == 1: + if verbose > 0: print('get_calib_data: use temperature calibration version 1') - backtime=back_DL[1] - backtime=backtime.tolist() - divertorpart="all" - TCT_dl=get_average_divertor_TC_for_camtemp(port,int(backtime-2e9),int(backtime+2e9),divertorpart) + backtime = back_DL[1] + backtime = backtime.tolist() + divertorpart = "all" + TCT_dl = get_average_divertor_TC_for_camtemp(port, int(backtime-2e9), int(backtime+2e9),divertorpart) if TCT_dl[0]: - refT=np.average(TCT_dl[2]) + refT = np.average(TCT_dl[2]) else: - TCT_dl=get_average_divertor_TC_for_camtemp(port,int(backtime-6e9),int(backtime+6e9),divertorpart) + TCT_dl = get_average_divertor_TC_for_camtemp(port, int(backtime-6e9), int(backtime+6e9),divertorpart) if TCT_dl[0]: - refT=np.average(TCT_dl[2]) + refT = np.average(TCT_dl[2]) else: raise Exception("get_calib_data: Unable to find thermocouple data") - elif T_version==2: - if verbose>0: + elif T_version == 2: + if verbose > 0: print('get_calib_data: use temperature calibration version 2') - frame=background.copy() - background=get_average_background_recangle(port,background) - gotit,waterin,waterout=get_cooling_water_temperature_by_time(t1) + frame = background.copy() + background = get_average_background_recangle(port, background) + gotit, waterin, waterout = get_cooling_water_temperature_by_time(t1) if gotit: - refT=waterout + refT = waterout else: - refT=28.5 - elif T_version==3: - if verbose>0: + refT = 28.5 + elif T_version == 3: + if verbose > 0: print('get_calib_data: use temperature calibration version 3, TEST CASE!!!') - frame=background.copy() - background=get_average_background_recangle(port,background) - refT=28.5 + frame = background.copy() + background = get_average_background_recangle(port, background) + refT = 28.5 # read LUT V3 (compensated BG signal and photons flux of BG) LUT_DL_wall = read_LUT_from_file(port, time, t_exp, 0.45, cfilter, verbose=verbose-1) if LUT_DL_wall[0]: - if verbose>0: + if verbose > 0: print('get_calib_data: got the wall LUT') LUT_wall = LUT_DL_wall[1] - index=IR_tools.find_nearest(LUT_wall[1],refT) - background=background-LUT_wall[0][index] - refT=0 - del LUT_DL_wall,LUT_wall + index = IR_tools.find_nearest(LUT_wall[1],refT) + background = background-LUT_wall[0][index] + refT = 0 + del LUT_DL_wall, LUT_wall LUT_DL = read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1) if LUT_DL[0]: - if verbose>0: + if verbose > 0: print('get_calib_data: got the V3 LUT') LUT = LUT_DL[1] del LUT_DL -# back_off=estimate_offset(port,program) +# back_off=estimate_offset(port, program) # background=(background-back_off)/(back_emissivity)+back_off else: raise Warning("get_calib_data: Unknown temperature calibration method") - return False,background,LUT,0,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error +# return False, background, LUT, 0, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error # get bad pixels if np.max(badpixels)==0: - if Camera.split("_")[0]=="Infratec": - badpixels=find_badpixels(port,frame-background,offset,niterations=10,tolerance=10,plot_it=False,verbose=verbose-1)#find_badpixels(port,gain,offset) + if Camera.split("_")[0] == "Infratec": + badpixels=find_badpixels(port, frame-background, offset, niterations=10, tolerance=10, plot_it=False, verbose=verbose-1)#find_badpixels(port, gain, offset) else: - if verbose>0: - print(datetime.datetime.now(),"get_calib_data: Scanning for bad pixel") + if verbose > 0: + print(datetime.datetime.now(), "get_calib_data: Scanning for bad pixel") # initial list from config files init_bp_list = IR_tools.read_bad_pixels_from_file(port, time_ns=t1) # find more bad pixel - badpixels = find_badpixels(port, gain, offset, init_bp_list=init_bp_list, niterations=10, tolerance=10,verbose=verbose-1) - if verbose>0: + badpixels = find_badpixels(port, gain, offset, init_bp_list=init_bp_list, niterations=10, tolerance=10, verbose=verbose-1) + if verbose > 0: print('get_calib_data: NUC arrays ready') - return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error + return exist, background, LUT,refT, gain, offset, badpixels, t_exp, cfilter, gain_error, offset_error @@ -2653,7 +2687,7 @@ def find_badpixels(port, gain, offset, init_bp_list=None, niterations=3, FOV=get_FOV_mask(port) # take initial bad pixels into account if init_bp_list is not None: - if verbose>0: + if verbose > 0: print("find_badpixels: use {0} initial bad pixels from file".format(len(init_bp_list))) for pix in init_bp_list: try: @@ -2666,10 +2700,10 @@ def find_badpixels(port, gain, offset, init_bp_list=None, niterations=3, finished=False n=0 while (n<=niterations and not finished):#len(badlist)>=last_number): - badlist=IR_tools.find_outlier_pixels(gain*FOV,plot_it=plot_it,tolerance=tolerance) - gain=IR_tools.restore_bad_pixels(gain*FOV,badlist, verbose=verbose-1) - if verbose>0: - print("find_badpixels: number of found bad pixels: ",len(badlist)) + badlist=IR_tools.find_outlier_pixels(gain*FOV, plot_it=plot_it, tolerance=tolerance) + gain=IR_tools.restore_bad_pixels(gain*FOV, badlist, verbose=verbose-1) + if verbose > 0: + print("find_badpixels: number of found bad pixels: ", len(badlist)) n+=1 if len(badlist)>=last_number and n>2: finished=True @@ -2689,7 +2723,7 @@ def make_FOV_mask(port): """ points=valid_FOV_circle[port] - # Sekante errechnen für Kreis, Gerade ax+by=c, sonderfall: y=c, a=0,b=1 + # Sekante errechnen für Kreis, Gerade ax+by=c, sonderfall: y=c, a=0, b=1 # Kreis definiert durch (x-x0)²+(y-y0)²=r² # loesung: d=c-a*x0-b*y0=c-y0 # x1,2=x0+(ad+-b*sqrt(r²(a²+b²)-d²))/(a²+b²)=x0+-sqrt(r²-d²) @@ -2704,10 +2738,10 @@ def make_FOV_mask(port): else: exposure=9 cfilter=0 - da,time,back=download_background_by_program(port,"20171109.045",exposure,camera_filter=cfilter) + da, time, back=download_background_by_program(port, "20171109.045", exposure, camera_filter=cfilter) # fig = plt.figure() -# plt.imshow(back,vmin=np.average(back)-200,vmax=np.average(back)+500) -# inner_circle = mlt.patches.Circle((x0,y0), r1,color = 'r', fill = False) +# plt.imshow(back, vmin=np.average(back)-200, vmax=np.average(back)+500) +# inner_circle = mlt.patches.Circle((x0,y0), r1, color = 'r', fill = False) # ax = fig.gca() # ax.add_artist(inner_circle) if da: @@ -2715,61 +2749,61 @@ def make_FOV_mask(port): for y in range(len(back)): xs1=x0-np.sqrt(r1**2-(y-y0)**2) xs2=x0+np.sqrt(r1**2-(y-y0)**2) - for xi in range(0,int(xs1)): + for xi in range(0, int(xs1)): FOV[y][xi]=0 - for xi in range(int(xs2),np.shape(back)[1]): + for xi in range(int(xs2), np.shape(back)[1]): FOV[y][xi]=0 plt.figure() - plt.imshow(FOV*back,vmin=np.average(back)-500,vmax=np.average(back)+1500) - if config_path=="": - plt.imsave("AEF"+str(port)+"_FOV.png",FOV) + plt.imshow(FOV*back, vmin=np.average(back)-500, vmax=np.average(back)+1500) + if config_path == "": + plt.imsave("AEF"+str(port)+"_FOV.png", FOV) else: - plt.imsave(config_path+portcamdict['OP1.2a']['AEF'+str(port)]+"\\"+"AEF"+str(port)+"_FOV.png",FOV) + plt.imsave(config_path+portcamdict['OP1.2a']['AEF'+str(port)]+"\\"+"AEF"+str(port)+"_FOV.png", FOV) else: - raise Exception("make_FOV_mask: cannot find the background frame to create the FOV file",exposure,port) + raise Exception("make_FOV_mask: cannot find the background frame to create the FOV file", exposure, port) def get_FOV_mask(port): """ """ try: if config_path=="": - FOV=plt.imread("AEF"+str(port)+"_FOV.png")[:,:,0] + FOV=plt.imread("AEF"+str(port)+"_FOV.png")[:,:, 0] else: - FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(port)]+"\\"+"AEF"+str(port)+"_FOV.png")[:,:,0] + FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(port)]+"\\"+"AEF"+str(port)+"_FOV.png")[:,:, 0] except: make_FOV_mask(port) if config_path=="": - FOV=plt.imread("AEF"+str(port)+"_FOV.png")[:,:,0] + FOV=plt.imread("AEF"+str(port)+"_FOV.png")[:,:, 0] else: - FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(port)]+"\\"+"AEF"+str(port)+"_FOV.png")[:,:,0] + FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(port)]+"\\"+"AEF"+str(port)+"_FOV.png")[:,:, 0] FOV=(FOV>0.1)*1 return FOV -def get_background(port,image): +def get_background(port, image): """ """ try: # points=valid_background_rectangle[port] FOV=get_FOV_mask(port) - dummy=FOV*image#image[points[1]:points[3],points[0]:points[2]] + dummy=FOV*image#image[points[1]:points[3], points[0]:points[2]] return np.min(dummy[np.nonzero(dummy)]) except Exception as E: - print('get_background: Error! ',E) + print('get_background: Error! ', E) return 0 -def get_average_background_recangle(port,image): +def get_average_background_recangle(port, image): """ """ try: points=valid_background_rectangle[port] # FOV=get_FOV_mask(port) - dummy=image[points[1]:points[3],points[0]:points[2]]#FOV*image# + dummy=image[points[1]:points[3], points[0]:points[2]]#FOV*image# return np.min(dummy[np.nonzero(dummy)]) except Exception as E: - print('get_average_background_recangle: Error! ',E) + print('get_average_background_recangle: Error! ', E) return 0 -def estimate_offset(port,program,plot_it=False,verbose=0): +def estimate_offset(port, program, plot_it=False, verbose=0): """ """ explist=[1,2,3,4,5,6,7,8,9] @@ -2777,7 +2811,7 @@ def estimate_offset(port,program,plot_it=False,verbose=0): exlist=[] FOV=get_FOV_mask(port) for exposure in explist: - exist,time,frame=download_background_by_program(port,program,exposure) + exist, time, frame=download_background_by_program(port, program, exposure) if exist: meanlist.append(np.mean(frame[np.where(FOV==1)])) exlist.append(exposure) @@ -2787,22 +2821,22 @@ def estimate_offset(port,program,plot_it=False,verbose=0): slope, intercept, r_value, p_value, std_err = linregress(exlist,meanlist) if plot_it: plt.figure() - plt.plot(exlist,meanlist,'b*') - plt.plot(exlist,np.array(exlist)*slope+intercept,'k-') + plt.plot(exlist,meanlist, 'b*') + plt.plot(exlist, np.array(exlist)*slope+intercept, 'k-') plt.show() - if verbose>0: + if verbose > 0: print("estimate_offset: slope, intercept, r_value, p_value, std_err") - print(" ",slope, intercept, r_value, p_value, std_err) + print(" ", slope, intercept, r_value, p_value, std_err) return intercept else: return -1 -def download_heatflux_by_program(port,program,time_window=0,threads=1,testmode=False, - version=0,verbose=0,givealpha=False,request=True,moffset=0): +def download_heatflux_by_program(port, program, time_window=0, threads=1, testmode=False, + version=0, verbose=0, givealpha=False,request=True,moffset=0): """ port: string or integer program: string - time_window: integer,float or list/numpy array of interger/float, single value: t1 until so many seconds, two values, t1+ first value until t1+ second value + time_window: integer, float or list/numpy array of interger/float, single value: t1 until so many seconds, two values, t1+ first value until t1+ second value threads: integer, number of threads for parallel download testmode: True to load data from testarchive version: integer, version number for 0 the highest version will be used @@ -2816,7 +2850,7 @@ def download_heatflux_by_program(port,program,time_window=0,threads=1,testmode=F exist, _, t1, t6 = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if not exist: print("download_heatflux_by_program: Error! program not found") - return False,0,-1 + return False, 0,-1 if time_window==0: tstart=t1 @@ -2830,7 +2864,7 @@ def download_heatflux_by_program(port,program,time_window=0,threads=1,testmode=F tstart=int(t1) tend=int(t1+time_window[0]*1e9) except Exception as E: - raise Exception("download_heatflux_by_program: an Exception occured interpretating the time_window: ",E) + raise Exception("download_heatflux_by_program: an Exception occured interpretating the time_window: ", E) elif type(time_window)==int or type(time_window)==float: tstart=int(t1) tend=int(t1+time_window*1e9) @@ -2838,14 +2872,14 @@ def download_heatflux_by_program(port,program,time_window=0,threads=1,testmode=F raise Exception("download_heatflux_by_program: given time_window type is not supported") if givealpha: tstart=tstart-1.1e9 - return download_heatflux_by_times(port=port,tstart=tstart,tend=tend,time_window=0, - testmode=testmode,version=version,verbose=verbose, + return download_heatflux_by_times(port=port, tstart=tstart, tend=tend, time_window=0, + testmode=testmode, version=version, verbose=verbose, request=request,moffset=moffset) -def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode=False, - version=0,verbose=0,request=True,moffset=0): +def download_heatflux_by_times(port, tstart, tend, time_window=0, threads=1, testmode=False, + version=0, verbose=0,request=True,moffset=0): """ port: string or integer tstart: int64 timestamp in ns from which point the download should start @@ -2854,24 +2888,24 @@ def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode testmode: True to load data from testarchive version: integer, version number for 0 the highest version will be used verbose: interger, level of outputs - returns found,time,images + returns found, time, images """ if testmode: base=testarchivepath else: base=archivepath - OP=IR_tools.get_OP_by_time(time_ns=tstart) + OP = IR_tools.get_OP_by_time(time_ns=tstart) if type(port) is str: - try: - camera=portcamdict[OP][port] - if camera.split("_")[0]=='FLIR': - FLIR=True - else: - FLIR=False - port=int(port.split("AEF")[1]) - goon=True - except Exception as E: - raise Exception("download_heatflux_by_times: unknown Port!"+E) + try: + camera=portcamdict[OP][port] + if camera.split("_")[0]=='FLIR': + FLIR=True + else: + FLIR=False + port=int(port.split("AEF")[1]) + goon=True + except Exception as E: + raise Exception("download_heatflux_by_times: unknown Port!"+E) elif isinstance(port, (int, np.integer, float, np.float)): FLIR=False goon=True @@ -2882,72 +2916,78 @@ def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode raise Exception("download_heatflux_by_times: the given port is neither a number or a valid String!") else: if FLIR: - if verbose>0: + if verbose > 0: print("download_heatflux_by_times: FLIR heatflux is at the moment unsupported!") - return False,0,-1 + return False, 0,-1 else:## Okay QRT is requested. Maybe there is data available, maybe not, lets find out - if version==0: - version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"heatflux_DATASTREAM",project=project_ana,testmode=testmode,t_from=tstart,t_to=tend) + if version == 0: + version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"heatflux_DATASTREAM", project=project_ana, testmode=testmode, t_from=tstart, t_to=tend) larchivepath=base+project_ana+"/"+portpathdict[OP]["AEF"+str(port)]+"heatflux_DATASTREAM/V"+str(version)+"/0/heatflux" ### test for the testsample### -# if version==0: -# version=get_latest_version("QRT_IRCAM_Test/AEF"+str(port)+"_heatflux_Test_2_DATASTREAM",project=project_ana,testmode=testmode) +# if version == 0: +# version = get_latest_version("QRT_IRCAM_Test/AEF"+str(port)+"_heatflux_Test_2_DATASTREAM", project=project_ana, testmode=testmode) # larchivepath=base+project_ana+"/QRT_IRCAM_Test/AEF"+str(port)+"_heatflux_Test_2_DATASTREAM/V"+str(version)+"/0/heatflux" ### end of testsample modifikation### - if verbose>0: - print(datetime.datetime.now(),"download_heatflux_by_times: heat flux download started") + if verbose > 0: + print(datetime.datetime.now(), "download_heatflux_by_times: heat flux download started") if False: ## does not work, the png pictures are interpretated as colors, values are lost - exist,time,frames=download_images_by_time_via_png(larchivepath,starttime=tstart,stoptime=tend,time_window=time_window,version=version,threads=threads,verbose=verbose-1) + exist, time, frames=download_images_by_time_via_png(larchivepath, starttime=tstart, stoptime=tend, time_window=time_window, version=version, threads=threads, verbose=verbose-1) else: - exist,time,frames=download_images_by_times(larchivepath,starttime=tstart,stoptime=tend,version=version,verbose=verbose-1) + exist, time, frames=download_images_by_times(larchivepath, starttime=tstart, stoptime=tend, version=version, verbose=verbose-1) if exist==False: now=datetime.datetime.now() - if verbose>0: - print(now,"download_heatflux_by_times: heat flux data is not available") + if verbose > 0: + print(now, "download_heatflux_by_times: heat flux data is not available") if request: try: programid=AKF_2.get_program_id(tstart) except Exception as E: - if verbose>0: - print('download_heatflux_by_times: Error! ',E) - programid=str(tstart) - cam_progs=IR_tools.get_work_list(heatflux_requestlist_path,typ='load') - cam_progs_ig,reasons=IR_tools.get_work_list(heatflux_requestlist_path,typ='ignore') - if (programid,"AEF"+str(port)) not in cam_progs and (programid,"AEF"+str(port)) not in cam_progs_ig: - f=open(heatflux_requestlist_path+str(now.year)+str(now.month+moffset)+"_q_requests.txt",'a') + if verbose > 0: + print('download_heatflux_by_times: Error! ', E) + programid=str(tstart) + try: + cam_progs=IR_tools.get_work_list(heatflux_requestlist_path, typ='load') + except: + cam_progs=[] + try: + cam_progs_ig,reasons=IR_tools.get_work_list(heatflux_requestlist_path, typ='ignore') + except: + cam_progs_ig=[] + if (programid, "AEF"+str(port)) not in cam_progs and (programid, "AEF"+str(port)) not in cam_progs_ig: + f=open(heatflux_requestlist_path+str(now.year)+str(now.month+moffset)+"_q_requests.txt", 'a') f.write(programid+"\tAEF"+str(port)+"\n") f.close() - if verbose>0: + if verbose > 0: print("download_heatflux_by_times: heat flux calculation request logged for automatic processing (within ca. 1 day)") - elif verbose>0: - if (programid,'AEF'+str(port)) in cam_progs: + elif verbose > 0: + if (programid, 'AEF'+str(port)) in cam_progs: print("download_heatflux_by_times: request exist already") else: - pid=cam_progs_ig.index((programid,'AEF'+str(port))) + pid=cam_progs_ig.index((programid, 'AEF'+str(port))) print("download_heatflux_by_times: request ignored, data not available, reason: {0}".format(reasons[pid])) - return exist,time,frames + return exist, time, frames -def download_heatflux_mapping_reference(timepoint=None,version=0,testmode=False, +def download_heatflux_mapping_reference(timepoint=None, version=0, testmode=False, get_thickness=False, verbose=0): """ return exist(boolean) and dictonary of the mappings informations """ if timepoint==None: - timepoint=int(TimeToNs([2017,9,26],[8,0,0,0])) + timepoint=int(TimeToNs([2017,9,26], [8, 0, 0, 0])) if testmode: base=testarchivepath else: base=archivepath -# OP=IR_tools.get_OP_by_time(time_ns=timepoint) - if version==0: - version=get_latest_version("QRT_IRCAM/Mapping_reference_DATASTREAM",project=project_ana,testmode=testmode,t_from=timepoint) +# OP = IR_tools.get_OP_by_time(time_ns=timepoint) + if version == 0: + version = get_latest_version("QRT_IRCAM/Mapping_reference_DATASTREAM", project=project_ana, testmode=testmode, t_from=timepoint) larchivepath=base+project_ana+"/"+"QRT_IRCAM/Mapping_reference_DATASTREAM/V"+str(version)+"/0/reference" ### test for the testsample### -# if version==0: -# version=get_latest_version("QRT_IRCAM_Test/Mapping_reference_Test_1_PARLOG",project=project_ana,testmode=testmode) +# if version == 0: +# version = get_latest_version("QRT_IRCAM_Test/Mapping_reference_Test_1_PARLOG", project=project_ana, testmode=testmode) # larchivepath=base+project_ana+"/QRT_IRCAM_Test/Mapping_reference_Test_1_DATASTREAM/V"+str(version)+"/0/reference" ### end of testsample ### - exist,time,frames=download_images_by_times(larchivepath,starttime=timepoint,stoptime=int(timepoint+1e9),version=version,verbose=verbose-1) + exist, time, frames=download_images_by_times(larchivepath, starttime=timepoint, stoptime=int(timepoint+1e9), version=version, verbose=verbose-1) mappings={} if exist: mappings['s']=frames[0] @@ -2955,7 +2995,7 @@ def download_heatflux_mapping_reference(timepoint=None,version=0,testmode=False, mappings['Y']=frames[2] mappings['Finger_X']=frames[3] mappings['Finger_Y']=frames[4] - mappings['Finger_ID']=(frames[5],"legend: first three digits are fingernumber,starting @0, last two are the profile number") + mappings['Finger_ID']=(frames[5], "legend: first three digits are fingernumber, starting @0, last two are the profile number") mappings['Target']=(frames[6],{1:"TM1-4h",2:"TM5-6h",3:"TM7-9h",4:"TM1-3v"}) # derive thickness of profile line (for integration) @@ -3020,7 +3060,7 @@ def download_heatflux_mapping_reference(timepoint=None,version=0,testmode=False, abs((y2_p-y1_p)*x_f - (x2_p-x1_p)*y_f + x2_p*y1_p - y2_p*x1_p) / np.sqrt((y2_p-y1_p)**2 + (x2_p-x1_p)**2) /2 + \ abs((y2_n-y1_n)*x_f - (x2_n-x1_n)*y_f + x2_n*y1_n - y2_n*x1_n) / np.sqrt((y2_n-y1_n)**2 + (x2_n-x1_n)**2) /2 mappings['Thickness'] = d - if verbose>0: + if verbose > 0: print("download_heatflux_mapping_reference: 'thickness' of profiles added to mapping reference dictionary") # shift vertical target in X and Y if it is centered on top of horizontal target @@ -3032,7 +3072,7 @@ def download_heatflux_mapping_reference(timepoint=None,version=0,testmode=False, y1 = Y[index_ver] X[index_ver] = np.cos(22.5/180*np.pi)*x1 + np.sin(22.5/180*np.pi)*y1 - 0.9 Y[index_ver] = -np.sin(22.5/180*np.pi)*x1 + np.cos(22.5/180*np.pi)*y1 + 0.7 - if verbose>0: + if verbose > 0: print("download_heatflux_mapping_reference: vertical target moved and rotated in 'X' and 'Y' for better plotting") # define target modules array @@ -3059,8 +3099,8 @@ def download_heatflux_mapping_reference(timepoint=None,version=0,testmode=False, return exist,mappings -def download_heatflux_scene_model_reference(port,timepoint=None,program=None, - version=0,testmode=False,verbose=0): +def download_heatflux_scene_model_reference(port, timepoint=None, program=None, + version=0, testmode=False, verbose=0): """ return exist(boolean) and dictonary of the mappings informations """ @@ -3069,7 +3109,7 @@ def download_heatflux_scene_model_reference(port,timepoint=None,program=None, elif type(port)==str: portnam=port if timepoint==None and program==None: - timepoint=int(TimeToNs([2017,8,28],[8,0,0,0])) + timepoint=int(TimeToNs([2017,8,28], [8, 0, 0, 0])) elif timepoint==None: # prog=get_program_from_PID(program) # if prog[0]: @@ -3082,15 +3122,15 @@ def download_heatflux_scene_model_reference(port,timepoint=None,program=None, base=testarchivepath else: base=archivepath -# OP=IR_tools.get_OP_by_time(time_ns=timepoint) - if version==0: - version=get_latest_version("QRT_IRCAM/"+str(portnam)+"_SceneModel_reference_PARLOG",project=project_ana,testmode=testmode,t_from=timepoint) +# OP = IR_tools.get_OP_by_time(time_ns=timepoint) + if version == 0: + version = get_latest_version("QRT_IRCAM/"+str(portnam)+"_SceneModel_reference_PARLOG", project=project_ana, testmode=testmode, t_from=timepoint) larchivepath=base+project_ana+"/"+"QRT_IRCAM/"+str(portnam)+"_SceneModel_reference_DATASTREAM/V"+str(version)+"/0/scene%20model%20reference" larchivepath_par=base+project_ana+"/QRT_IRCAM/"+str(portnam)+"_SceneModel_reference_PARLOG/V"+str(version)+"/parms/header/Pixel_X" ### test for the testsample### #============================================================================== -# if version==0: -# version=get_latest_version("QRT_IRCAM_Test/"+str(portnam)+"_SceneModel_reference_PARLOG",project=project_ana,testmode=testmode) +# if version == 0: +# version = get_latest_version("QRT_IRCAM_Test/"+str(portnam)+"_SceneModel_reference_PARLOG", project=project_ana, testmode=testmode) # larchivepath=base+project_ana+"/QRT_IRCAM_Test/"+str(portnam)+"_SceneModel_reference_DATASTREAM/V"+str(version)+"/0/scene%20model%20reference" # larchivepath_par=base+project_ana+"/QRT_IRCAM_Test/"+str(portnam)+"_SceneModel_reference_PARLOG/V"+str(version)+"/parms/header/Pixel_X" #============================================================================== @@ -3101,14 +3141,15 @@ def download_heatflux_scene_model_reference(port,timepoint=None,program=None, res.close() goon=True except urllib.error.URLError as e: - print('download_heatflux_scene_model_reference: Error! ',e,larchivepath_par+"/_signal.json?from="+str(timepoint)+"&upto="+str(timepoint)) + print('download_heatflux_scene_model_reference: Error! ', e, larchivepath_par+"/_signal.json?from="+str(timepoint)+"&upto="+str(timepoint)) goon=False except Exception as E: - print('download_heatflux_scene_model_reference: Error! ',E) + print('download_heatflux_scene_model_reference: Error! ', E) goon=False if goon: timepoint=signal_list['values'][0] - exist,time,frames=download_images_by_times(larchivepath,starttime=timepoint-10,stoptime=int(timepoint+1e8),version=version,verbose=verbose-1) + exist, time, frames = download_images_by_times(larchivepath, starttime=timepoint-10, stoptime=int(timepoint+1e8), version=version, verbose=verbose-1) + del time mappings={} if exist: mappings['Pixel_X']=frames[0] @@ -3116,103 +3157,103 @@ def download_heatflux_scene_model_reference(port,timepoint=None,program=None, mappings['CAD_X']=frames[2] mappings['CAD_Y']=frames[3] mappings['CAD_Z']=frames[4] - mappings['PFC']=(frames[5],"legend: https://wikis.ipp-hgw.mpg.de/W7X/images/9/9c/PFC2.pdf") + mappings['PFC']=(frames[5], "legend: https://wikis.ipp-hgw.mpg.de/W7X/images/9/9c/PFC2.pdf") return exist,mappings else: return False,{} -def give_finger_ID(profile,finger=None,verbose=0): +def give_finger_ID(profile, finger=None, verbose=0): """ profile: string, integer or float, string: "TM3h_5_5" or "1605" or "16.05" or "5", similar for float and integer. single number only if the finger is given! finger: string or integer, string "TM3h_5" or "16" or 16 or 1600 or None """ ### offsets={ - "tm1h":0,"tm2h":6,"tm3h":13,"tm4h":21,"tm5h":29,"tm6h":53,"tm7h":77,"tm8h":83,"tm9h":95,"tm1v":107,"tm2v":117,"tm3v":127} + "tm1h":0, "tm2h":6, "tm3h":13, "tm4h":21, "tm5h":29, "tm6h":53, "tm7h":77, "tm8h":83, "tm9h":95, "tm1v":107, "tm2v":117, "tm3v":127} ### go throw the different cases, case: only one if given - fingerID=-1 - if finger==None:##combinations where only profile is given - if type(profile)==int: - fingerID=profile - success=True - elif type(profile)==float: + fingerID = -1 + if finger == None:##combinations where only profile is given + if type(profile) == int: + fingerID = profile + success = True + elif type(profile) == float: fingerID=int(profile*100) - success=True - elif type(profile)==str: + success = True + elif type(profile) == str: profile=profile.lower() if "." in profile: - fingerID=int(float(profile)*100) - success=True + fingerID = int(float(profile)*100) + success = True elif "tm" in profile: ## special case with TM have to be included - prof=profile.split("_") - offset=offsets[prof[0]] - fingerID=(offset+int(prof[1]))*100+int(prof[2]) - success=True + prof = profile.split("_") + offset = offsets[prof[0]] + fingerID = (offset+int(prof[1]))*100+int(prof[2]) + success = True else: - fingerID=int(profile) - success=True + fingerID = int(profile) + success = True else: - success=False - if verbose>0: + success = False + if verbose > 0: print("give_finger_ID: type of profile is not supported!") else:#cases with both are given - if type(finger)==str: - finger=finger.lower() + if type(finger) == str: + finger = finger.lower() if "tm" in finger: - fing=finger.split("_") - offset=offsets[fing[0]] - finger=offset+int(fing[1]) + fing = finger.split("_") + offset = offsets[fing[0]] + finger = offset+int(fing[1]) else: - finger=int(finger) + finger = int(finger) #finger is now an int - if finger%100==0: #check whether the finger format is 1600 or equal to it, reduce it to 16 or equal - finger=int(finger/100) - if type(profile)==int and type(finger)==int:#simple case - if finger<138 and profile<18: - success=True - fingerID=finger*100+profile + if finger%100 == 0: #check whether the finger format is 1600 or equal to it, reduce it to 16 or equal + finger = int(finger/100) + if type(profile) == int and type(finger) == int:#simple case + if finger < 138 and profile < 18: + success = True + fingerID = finger*100+profile else: - success=False - if verbose>0: + success = False + if verbose > 0: print("give_finger_ID: given finger or profile is out of valid range! (finger 0-137, profiles 0-17)") ### combinations left: str-str, float-str, float-int, str-int, int-str - elif type(profile)==str or type(profile)==float:#combinations str-str and str-int + elif type(profile) == str or type(profile) == float:#combinations str-str and str-int try:### str: by logic only integer should be left if input is a valid one, float: in theory for a valid no further digits should be there - if type(profile)==float: - if profile<1.0: - profile=profile*100 - fingerID=finger*100+int(profile) - success=True + if type(profile) == float: + if profile < 1.0: + profile = profile*100 + fingerID = finger*100+int(profile) + success = True except: - success=False - if verbose>0: + success = False + if verbose > 0: print("give_finger_ID: profile and finger defined, profile is not in a valid format!") - elif type(profile)==int: - if profile<18: - fingerID=finger*100+profile - success=True + elif type(profile) == int: + if profile < 18: + fingerID = finger*100+profile + success = True else: - success=False - if verbose>0: + success = False + if verbose > 0: print("give_finger_ID: profile is out of valid range (0-17)") else: - success=False - if verbose>0: + success = False + if verbose > 0: print("give_finger_ID: type of profile is not supported!") if success:#last test if fingerID%100<18 and fingerID//100<138 and not fingerID==-1: - success=True + success = True else: - success=False - if verbose>0: + success = False + if verbose > 0: print("give_finger_ID: invalid fingerID found. Please check your input.") - return success,fingerID + return success, fingerID -def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstart=None, - tend=None,testmode=False,version=0,verbose=0): +def get_heatflux_profile(port, profile, finger=None, timepoint=1, program=None, tstart=None, + tend=None, testmode=False, version=0, verbose=0): """ - returns exist,time,s in m, heat flux in W/m2 + returns exist, time, s in m, heat flux in W/m2 parameters: port: string or integer @@ -3220,25 +3261,25 @@ def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstar finger: string or integer, string "TM3h_5" or "16" """ ### checking whether input is valid and whether the download can be started - if program==None and tstart==None: + if program == None and tstart == None: raise Exception("get_heatflux_profile: No Program given, please specify the program you want heatflux from") else: ### some informations about the program are given, is the timepoint given in a usefull way? - if timepoint==None or (type(timepoint)!=int and type(timepoint)!=float): - if verbose>0: + if timepoint == None or (type(timepoint) != int and type(timepoint) != float): + if verbose > 0: print("get_heatflux_profile: timepoint is not given in a usefull way, the profile have to be requested for location and time and program!") - return False,0,0,0 + return False, 0, 0, 0 ### is the location given? possible combinations for profile have to be checked - checkprof=give_finger_ID(profile,finger) + checkprof = give_finger_ID(profile, finger) if checkprof[0]: - fingerID=checkprof[1] - inputcheck=True + fingerID = checkprof[1] + inputcheck = True else: - if verbose>0: - print(datetime.datetimenow(),"get_heatflux_profile: given Finger and Profile combination is not understood!") - return False,0,0,0 + if verbose > 0: + print(datetime.datetime.now(), "get_heatflux_profile: given Finger and Profile combination is not understood!") + return False, 0, 0, 0 ### lets find out which timepoint or timeintervall is requested, to know the OP, only needed for OP2 changes, but who knows - if program!=None: + if program != None: # prog=get_program_from_PID(program) # if prog[0]: # t1=prog[1]['trigger']['1'][0] @@ -3247,236 +3288,265 @@ def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstar tstart=t1+int((timepoint-0.02)*1e9) tend=t1+int((timepoint+0.02)*1e9) else: - if verbose>0: + if verbose > 0: print("get_heatflux_profile: program not found") - return False,0,0,0 + return False, 0, 0, 0 else: - t1=tstart - if tend==None: - program=AKF_2.get_program_id(t1) + t1 = tstart + if tend == None: + program = AKF_2.get_program_id(t1) # prog=get_program_from_PID(program) # if prog[0]: # tref=prog[1]['trigger']['6'][0] exist, _, _, tref = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if exist: - tend=tref+int((timepoint+0.02)*1e9) - OP=IR_tools.get_OP_by_time(time_ns=t1) + tend = tref+int((timepoint+0.02)*1e9) + OP = IR_tools.get_OP_by_time(time_ns=t1) ### is the port given? if type(port) is str: try: camera=portcamdict[OP][port] if camera.split("_")[0]=='FLIR': - FLIR=True + FLIR = True else: - FLIR=False - port=int(port.split("AEF")[1]) - goon=True + FLIR = False + port = int(port.split("AEF")[1]) + goon = True except Exception as E: raise Warning("get_heatflux_profile: unknown Port!"+E) - return False,0,0,0 +# return False, 0, 0, 0 elif isinstance(port, (int, np.integer, float, np.float)): - FLIR=False - goon=True - port=port + FLIR = False + goon = True else: - goon=False + goon = False if not goon or FLIR: raise Exception("get_heatflux_profile: Unknown port, FLIR data is unsupported at the moment") else: - ### if the program goes up to here, time okay, finger okay,port okay, lets download it - exist,timo,frames=download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode=testmode,version=version,verbose=verbose-1) + ### if the program goes up to here, time okay, finger okay, port okay, lets download it + exist, timo, frames=download_heatflux_by_times(port, tstart, tend, time_window=0, threads=1, testmode=testmode, version=version, verbose=verbose-1) if exist: - return extract_heatflux_profile_from_DL(time=(np.asarray(timo)-t1)/1e9,images=frames,profile=int(fingerID%100),finger=int(fingerID/100),time_window=timepoint,inputchecked=inputcheck,verbose=verbose-1) + return extract_heatflux_profile_from_DL(time=(np.asarray(timo)-t1)/1e9, images=frames, profile=int(fingerID%100), finger=int(fingerID/100), time_window=timepoint, inputchecked=inputcheck, verbose=verbose-1) else: - if verbose>0: + if verbose > 0: print("get_heatflux_profile: heatflux data could not be downloaded") - return False,0,0,0 + return False, 0, 0, 0 -def extract_heatflux_profile_from_DL(time,images,profile,finger=None,time_window=1, - inputchecked=False,verbose=0,testmode=False): +def extract_heatflux_profile_from_DL(time, images, profile, finger=None, time_window=1, + inputchecked=False, verbose=0, testmode=False): """ - return exist,time,s in m, heat flux in W/m2 + return exist, time, s in m, heat flux in W/m2 time_point default (None) returns entire shot. """ - goon=True - if time_window==None: - Tid_start=0 - Tid_end=len(time) - elif type(time_window)==int or type(time_window)==float: + goon = True + if time_window == None: + Tid_start = 0 + Tid_end = len(time) + elif type(time_window) == int or type(time_window) == float: try: - Tid_start=IR_tools.find_nearest(time,time_window) + Tid_start=IR_tools.find_nearest(time, time_window) Tid_end=Tid_start+1 except Exception as E: - goon=False - elif type(time_window)==list or type(time_window)==np.ndarray and len(time_window)>0: + goon = False + elif type(time_window) == list or type(time_window) == np.ndarray and len(time_window)>0: try: - Tid_start=IR_tools.find_nearest(time,time_window[0]) + Tid_start = IR_tools.find_nearest(time, time_window[0]) except Exception as E: - goon=False + goon = False if len(time_window)>1: try: - Tid_end=IR_tools.find_nearest(time,time_window[1]) + Tid_end = IR_tools.find_nearest(time, time_window[1]) except Exception as E: - goon=False + goon = False else: - Tid_end=Tid_start+1 + Tid_end = Tid_start+1 else: goon=False - E="unknown Case" + E = "unknown Case" if not goon: - if verbose>0: - print('extract_heatflux_profile_from_DL: ',E) - return False,0,0,0 + if verbose > 0: + print('extract_heatflux_profile_from_DL: ', E) + return False, 0, 0, 0 else: if not inputchecked: - checkprof=give_finger_ID(profile,finger) + checkprof = give_finger_ID(profile, finger) if checkprof[0]: - fingerID=checkprof[1] + fingerID = checkprof[1] else: - return False,0,0,0 + return False, 0, 0, 0 else: - fingerID=finger*100+profile + fingerID = finger*100+profile ### okay finderID is fine, data is given, lets get the mapping - exist,mapping=download_heatflux_mapping_reference(testmode=testmode, verbose=verbose-1) + exist ,mapping = download_heatflux_mapping_reference(testmode=testmode, verbose=verbose-1) if exist: - Fingermap=np.asarray(mapping['Finger_ID'][0],dtype=np.int) - locs=np.where(Fingermap==fingerID) - S=mapping['s'][locs] - q=np.asarray([images[i][locs] for i in range(Tid_start,Tid_end)]) - return True,time[Tid_start:Tid_end],S,q + Fingermap = np.asarray(mapping['Finger_ID'][0],dtype=np.int) + locs = np.where(Fingermap==fingerID) + S = mapping['s'][locs] + q = np.asarray([images[i][locs] for i in range(Tid_start, Tid_end)]) + return True, time[Tid_start:Tid_end], S, q else: - if verbose>0: + if verbose > 0: print("extract_heatflux_profile_from_DL: mapping was not found, cannot extract the profile") - return False,0,0,0 + return False, 0, 0, 0 -def extract_temperature_profile_from_DL(port,time,images,profile,finger=None,time_window=1, - inputchecked=False,verbose=0,reference_time=0): +def extract_temperature_profile_from_DL(port, time, images, profile, finger=None, + time_window=1, inputchecked=False, + verbose=0, reference_time=0): """ - return exist,time,s in m, heat flux in W/m2 + + INPUT + ------ + port: int or str + the camera port as number or full string (e.g. 'AEF10') + return exist, time, s in m, heat flux in W/m2 time_point default (None) returns entire shot. """ - goon=True - if time_window==None: - Tid_start=0 - Tid_end=len(time) - elif type(time_window)==int or type(time_window)==float: + goon = True + if time_window == None: + Tid_start = 0 + Tid_end = len(time) + elif type(time_window) == int or type(time_window) == float: try: - Tid_start=IR_tools.find_nearest(time,time_window) - Tid_end=Tid_start+1 + Tid_start = IR_tools.find_nearest(time, time_window) + Tid_end = Tid_start+1 except Exception as E: goon=False - elif type(time_window)==list or type(time_window)==np.ndarray and len(time_window)>0: + elif type(time_window) == list or type(time_window) == np.ndarray and len(time_window) > 0: try: - Tid_start=IR_tools.find_nearest(time,time_window[0]) + Tid_start = IR_tools.find_nearest(time, time_window[0]) except Exception as E: - goon=False + goon = False if len(time_window)>1: try: - Tid_end=IR_tools.find_nearest(time,time_window[1]) + Tid_end = IR_tools.find_nearest(time, time_window[1]) except Exception as E: - goon=False + goon = False else: - Tid_end=Tid_start+1 + Tid_end = Tid_start+1 else: - goon=False - E="unknown Case" + goon = False + E = "unknown Case" if not goon: - if verbose>0: - print('extract_temperature_profile_from_DL: ',E) - return False,0,0,0 + if verbose > 0: + print('extract_temperature_profile_from_DL: ', E) + return False, 0, 0, 0 else: if not inputchecked: - checkprof=give_finger_ID(profile,finger) + checkprof = give_finger_ID(profile, finger) if checkprof[0]: - fingerID=checkprof[1] + fingerID = checkprof[1] else: - return False,0,0,0 + return False, 0, 0, 0 else: - fingerID=finger*100+profile + fingerID = finger*100+profile ### okay finderID is fine, data is given, lets get the mapping - exist,mapping=download_heatflux_mapping_reference() + exist, mapping = download_heatflux_mapping_reference() if exist: - Fingermap=np.asarray(mapping['Finger_ID'][0],dtype=np.int) - locs=np.where(Fingermap==fingerID) - S=mapping['s'][locs] - exist_s,scene=download_heatflux_scene_model_reference(port,timepoint=reference_time,verbose=verbose-1) + Fingermap = np.asarray(mapping['Finger_ID'][0],dtype=np.int) + locs = np.where(Fingermap==fingerID) + S = mapping['s'][locs] + exist_s, scene = download_heatflux_scene_model_reference(port, timepoint=reference_time, verbose=verbose-1) if exist_s: - X=scene['Pixel_X'][locs] - Y=scene['Pixel_Y'][locs] - q=np.asarray([images[i][np.asarray(Y,dtype=np.int),np.asarray(X,dtype=np.int)] for i in range(Tid_start,Tid_end)]) - return True,time[Tid_start:Tid_end],S,q + X = scene['Pixel_X'][locs] + Y = scene['Pixel_Y'][locs] + q = np.asarray([images[i][np.asarray(Y,dtype=np.int), np.asarray(X,dtype=np.int)] for i in range(Tid_start, Tid_end)]) + return True, time[Tid_start:Tid_end], S, q else: - if verbose>0: + if verbose > 0: print("extract_temperature_profile_from_DL: mapping was not found, cannot extract the profile") - return False,0,0,0 + return False, 0, 0, 0 -def download_divertor_load(port,targetmodule=None,program=None,tstart=None,tend=None, - version=0,verbose=0,testmode=False,request=True): - """ +def download_divertor_load(port, targetmodule=None, program=None, tstart=None, + tend=None, version=0, verbose=0, testmode=False, + request=True): + """ + INPUT + ------ + port: int or str + the port description for the camera as number or full string (e.g. 'AEF10') + targetmodule: str or list/ndarray of str, optional + name or description of the target module, nothing wil return total load + program: str + program id as str in the format yyyymmdd.pid + tstart: uint64 + + tend: uint64 + + version: int + + verbose: int + + testmode: + + request: boolean, optional, default True + RESULT + ------ + exist: boolean + """ TMchadict={ - 'ALL':(0,1,'divertor_total_load'),'FULL':(0,1,'divertor_total_load'), - 'TM1H':(2,3,'TM1h_load'),'1H':(2,3,'TM1h_load'), - 'TM2H':(4,5,'TM2h_load'),'2H':(4,5,'TM2h_load'), - 'TM3H':(6,7,'TM3h_load'),'3H':(6,7,'TM3h_load'), - 'TM4H':(8,9,'TM4h_load'),'4H':(8,9,'TM4h_load'), - 'TM5H':(10,11,'TM5h_load'),'5H':(10,11,'TM5h_load'), - 'TM6H':(12,13,'TM6h_load'),'6H':(12,13,'TM6h_load'), - 'TM7H':(14,15,'TM7h_load'),'7H':(14,15,'TM7h_load'), - 'TM8H':(16,17,'TM8h_load'),'8H':(16,17,'TM8h_load'), - 'TM9H':(18,19,'TM9h_load'),'9H':(18,19,'TM9h_load'), - 'TM1V':(20,21,'TM1v_load'),'1V':(20,21,'TM1v_load'), - 'TM2V':(22,23,'TM2v_load'),'2V':(22,23,'TM2v_load'), - 'TM3V':(24,25,'TM3v_load'),'3V':(24,25,'TM3v_load') + 'ALL':(0,1, 'divertor_total_load'), 'FULL':(0,1, 'divertor_total_load'), + 'TM1H':(2,3, 'TM1h_load'), '1H':(2,3, 'TM1h_load'), + 'TM2H':(4,5, 'TM2h_load'), '2H':(4,5, 'TM2h_load'), + 'TM3H':(6,7, 'TM3h_load'), '3H':(6,7, 'TM3h_load'), + 'TM4H':(8,9, 'TM4h_load'), '4H':(8,9, 'TM4h_load'), + 'TM5H':(10,11, 'TM5h_load'), '5H':(10,11, 'TM5h_load'), + 'TM6H':(12,13, 'TM6h_load'), '6H':(12,13, 'TM6h_load'), + 'TM7H':(14,15, 'TM7h_load'), '7H':(14,15, 'TM7h_load'), + 'TM8H':(16,17, 'TM8h_load'), '8H':(16,17, 'TM8h_load'), + 'TM9H':(18,19, 'TM9h_load'), '9H':(18,19, 'TM9h_load'), + 'TM1V':(20,21, 'TM1v_load'), '1V':(20,21, 'TM1v_load'), + 'TM2V':(22,23, 'TM2v_load'), '2V':(22,23, 'TM2v_load'), + 'TM3V':(24,25, 'TM3v_load'), '3V':(24,25, 'TM3v_load') } #interpretation of the targetmodule input, string or List of strings, numbers does not make sense here? - if targetmodule==None: - datcha=0 - ercha=1 + if targetmodule == None: + datcha = 0 + ercha = 1 datcha_name='divertor_total_load' elif type(targetmodule)== list or type(targetmodule)==np.ndarray:## okay maybe more than one targetmodule is requested - datcha=[] - datcha_name=[] - ercha=[] + datcha = [] + datcha_name = [] + ercha = [] #okay now we have to find out which targetmodules are requested for ele in targetmodule: - dummy=str(ele).upper() + dummy = str(ele).upper() try: dummy=TMchadict[dummy] datcha.append(dummy[0]) ercha.append(dummy[1]) datcha_name.append(dummy[2]) except: - if verbose>0: - print("cannot interpretate",ele,"as a targetmodule, skip it") - elif type(targetmodule)==str: - targetmodule=targetmodule.upper() - dummy=TMchadict[targetmodule] - datcha=dummy[0] - datcha_name=dummy[2] - ercha=dummy[1] + if verbose > 0: + print("cannot interpretate", ele, "as a targetmodule, skip it") + elif type(targetmodule) == str: + targetmodule = targetmodule.upper() + dummy = TMchadict[targetmodule] + datcha = dummy[0] + datcha_name = dummy[2] + ercha = dummy[1] else:# - if verbose>0: + if verbose > 0: print("input for targetmodule invalid!") - return False,0,-1,-1 + return False, 0,-1,-1 # target module known and channels are known, time to know the time - if program==None and tstart==None: - if verbose>0: + if program == None and tstart == None: + if verbose > 0: print("input for program and time invalid!") - return False,0,-1,-1 - elif program!=None: + return False, 0,-1,-1 + elif program != None: # prog=get_program_from_PID(program) # if prog[0]: # tstart=prog[1]['trigger']['1'][0] # tend=prog[1]['trigger']['6'][0] # else: - exist, _, tstart, tend = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) - if not exist: - print("download_divertor_load: Error! program not found") - return False,0,-1,-1 + exist, _, tstart, tend = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) + if not exist: + print("download_divertor_load: Error! program not found") + return False, 0,-1,-1 else: - if tend==None: + if tend == None: program=AKF_2.get_program_id(tstart) # prog=get_program_from_PID(program) # if prog[0]: @@ -3484,88 +3554,112 @@ def download_divertor_load(port,targetmodule=None,program=None,tstart=None,tend= exist, _, _, tend = get_trigger_from_PID(program, port, testmode, verbose=verbose-1) if not exist: print("download_divertor_load: Error! end trigger not found") - return False,0,-1,-1 + return False, 0,-1,-1 #channels known, time known, okay lets get the data # tstart=int(tstart-100e6) - OP=IR_tools.get_OP_by_time(time_ns=tstart) + OP = IR_tools.get_OP_by_time(time_ns=tstart) if testmode: - base=testarchivepath + base = testarchivepath else: - base=archivepath - if type(port)==int: - port="AEF"+str(port) - if version==0: - version=get_latest_version(portpathdict[OP][str(port)]+"loads_DATASTREAM",project=project_ana,t_from=tstart,t_to=tend,testmode=testmode) - larchivepath=base+project_ana+"/"+portpathdict[OP][str(port)]+"loads_DATASTREAM/V"+str(version) - if type(datcha)==list:#case of several requested channels - load=[] - error=[] - exist=True + base = archivepath + if type(port) == int: + port = "AEF"+str(port) + if version == 0: + version = get_latest_version(portpathdict[OP][str(port)]+"loads_DATASTREAM", project=project_ana, t_from=tstart, t_to=tend, testmode=testmode) + larchivepath = base+project_ana+"/"+portpathdict[OP][str(port)]+"loads_DATASTREAM/V"+str(version) + if type(datcha) == list:#case of several requested channels + load = [] + error = [] + exist = True for i in range(len(datcha)): - data=read_restdb(larchivepath+"/"+str(datcha[i])+"/"+datcha_name[i]+"/_signal.json?from="+str(tstart)+"&upto="+str(tend)) - err=read_restdb(larchivepath+"/"+str(ercha[i])+"/"+datcha_name[i]+"_error/_signal.json?from="+str(tstart)+"&upto="+str(tend)) - if i==0: + data = read_restdb(larchivepath+"/"+str(datcha[i])+"/"+datcha_name[i]+"/_signal.json?from="+str(tstart)+"&upto="+str(tend)) + err = read_restdb(larchivepath+"/"+str(ercha[i])+"/"+datcha_name[i]+"_error/_signal.json?from="+str(tstart)+"&upto="+str(tend)) + if i == 0: if data[0]: - time=data[1] + time = data[1] else: - time=0 - exist=False + time = 0 + exist = False if data[0]: load.append(data[2]) if err[0]: error.append(err[2]) - if len(datcha)!=len(load): - exist=False + if len(datcha) != len(load): + exist = False else:# case for a single channel - data=read_restdb(larchivepath+"/"+str(datcha)+"/"+datcha_name+"/_signal.json?from="+str(tstart)+"&upto="+str(tend)) - err=read_restdb(larchivepath+"/"+str(ercha)+"/"+datcha_name+"_error/_signal.json?from="+str(tstart)+"&upto="+str(tend)) + data = read_restdb(larchivepath+"/"+str(datcha)+"/"+datcha_name+"/_signal.json?from="+str(tstart)+"&upto="+str(tend)) + err = read_restdb(larchivepath+"/"+str(ercha)+"/"+datcha_name+"_error/_signal.json?from="+str(tstart)+"&upto="+str(tend)) if data[0] and err[0]: - exist=True - time=data[1] - load=data[2] - error=err[2] + exist = True + time = data[1] + load = data[2] + error = err[2] else: - exist=False - time=0 - load=-1 - error=-1 - if exist==False: - now=datetime.datetime.now() - if verbose>0: - print(now,"download_divertor_load: divertor_load data is not available, creating request") + exist = False + time = 0 + load = -1 + error = -1 + if exist == False: + now = datetime.datetime.now() + if verbose > 0: + print(now, "download_divertor_load: divertor_load data is not available, creating request") if request: try: programid=AKF_2.get_program_id(tstart) except Exception as E: - if verbose>0: - print('download_divertor_load: Error! ',E) + if verbose > 0: + print('download_divertor_load: Error! ', E) programid=str(tstart) - cam_progs=IR_tools.get_work_list(heatflux_requestlist_path,typ='load') - cam_progs_ig,reasons=IR_tools.get_work_list(heatflux_requestlist_path,typ='ignore') - if (programid,str(port)) not in cam_progs and (programid,str(port)) not in cam_progs_ig: - f=open(heatflux_requestlist_path+str(now.year)+str(now.month)+"_load_requests.txt",'a') + cam_progs=IR_tools.get_work_list(heatflux_requestlist_path, typ='load') + cam_progs_ig,reasons=IR_tools.get_work_list(heatflux_requestlist_path, typ='ignore') + if (programid, str(port)) not in cam_progs and (programid, str(port)) not in cam_progs_ig: + f = open(heatflux_requestlist_path+str(now.year)+str(now.month)+"_load_requests.txt", 'a') f.write(programid+"\t"+str(port)+"\n") f.close() - if verbose>0: - print(now,"download_divertor_load: request created") - elif verbose>0: - if (programid,str(port)) in cam_progs: - print(now,"download_divertor_load: request exist already") + if verbose > 0: + print(now, "download_divertor_load: request created") + elif verbose > 0: + if (programid, str(port)) in cam_progs: + print(now, "download_divertor_load: request exist already") else: - pid=cam_progs_ig.index((programid,str(port))) - print(now,"download_divertor_load: request ignored, data not availabel, reason: {0}".format(reasons[pid])) - return exist,time,load,error + pid = cam_progs_ig.index((programid, str(port))) + print(now, "download_divertor_load: request ignored, data not available, reason: {0}".format(reasons[pid])) + return exist, time, load, error -def download_scene_model(port,program=None,timepoint=None,version=0,testmode=False,verbose=0): - """ - """ +def download_scene_model(port, program=None, timepoint=None, version=0, testmode=False, verbose=0): + ''' downloading the scene model, provided by F. Pisano + + INPUT + ----- + port: integer, float, string or list of int, float or strings + array of heat fluxes from THEODOR on the profiles defined + in the IR mapping; can be 2D (one divertor), or 3D (multiple divertor modules) + program : string, optional + Program ID of an W7-X Program in form of "20160224.025" to indicate the validation window + timepoint: int64, optional + nanosecond timestamp to indicate the validation window of the scene model + version: integer, optional + testmode: boolean, optional + verbose: integer, optional + RESULT + ------ + exist: boolean + indicates whether data was found (True) or not (False) + scene_model: dict + dictionary of the different images in the scene model (see keys for describtion) + + + NOTES + ----- + + ''' if type(port)==int or type(port)==float: portnam="AEF"+str(port) elif type(port)==str: portnam=port if program==None and timepoint==None: dati=datetime.datetime.now() - timepoint=int(TimeToNs([dati.year,dati.month,dati.day],[dati.hour,dati.minute,dati.second,dati.microsecond])) + timepoint=int(TimeToNs([dati.year,dati.month,dati.day], [dati.hour,dati.minute,dati.second,dati.microsecond])) elif timepoint==None: # prog=get_program_from_PID(program) # if prog[0]: @@ -3578,8 +3672,8 @@ def download_scene_model(port,program=None,timepoint=None,version=0,testmode=Fal base=testarchivepath else: base=archivepath - if version==0: - version=get_latest_version("QRT_IRCAM/"+str(portnam)+"_scene_model_PARLOG",project=project,testmode=False,t_from=timepoint) + if version == 0: + version = get_latest_version("QRT_IRCAM/"+str(portnam)+"_scene_model_PARLOG", project=project, testmode=False, t_from=timepoint) larchivepath=base+project+"/"+"QRT_IRCAM/"+str(portnam)+"_scene_model_DATASTREAM/V"+str(version)+"/0/"+str(portnam)+"_scene_model" larchivepath_par=base+project+"/QRT_IRCAM/"+str(portnam)+"_scene_model_PARLOG/V"+str(version) @@ -3589,12 +3683,13 @@ def download_scene_model(port,program=None,timepoint=None,version=0,testmode=Fal res.close() goon=True except urllib.error.URLError as e: - print('download_scene_model: Error! ',e) + print('download_scene_model: Error! ', e) goon=False if goon: timepoint=signal_list['values'][0]['meta-data']['timestamp'] height=signal_list['values'][0]['meta-data']['height'] - exist,time,frames=download_images_by_times(larchivepath,starttime=timepoint-10,stoptime=int(timepoint+1e8),version=version,verbose=verbose-1) + exist, time, frames = download_images_by_times(larchivepath, starttime=timepoint-10, stoptime=int(timepoint+1e8), version=version, verbose=verbose-1) + del time stacked_image = np.array(frames[0], dtype='float64') channels = np.array(np.vsplit(stacked_image, stacked_image.shape[0]/height)) scene_model = { @@ -3610,9 +3705,9 @@ def download_scene_model(port,program=None,timepoint=None,version=0,testmode=Fal "y": channels[9], "z": channels[10] } - return exist,scene_model + return exist, scene_model else: - return False,[0] + return False, [0] #%% general download functions """ @@ -3623,10 +3718,15 @@ the following functions are copied from W7xrest.read_restdb to remove this depen def read_restdb_old(request_url): """ Reads JSON data from W7-X REST API - Returns: - valid: access ok - t: numpy-array of time - signal: numpy-array of requested signals + + Returns + ------ + valid: bolean + access ok + t: numpy-array + numpy-array of time + signal: numpy-array + numpy-array of requested signals by H. Thomsen """ try: @@ -3648,11 +3748,16 @@ def read_restdb_old(request_url): def read_restdb(request_url): """ - Reads JSON data from W7-X REST API + Reads JSON data from W7-X REST API. Signals are converted into double + Returns: - valid: access ok - t: numpy-array of time - signal: numpy-array of requested signals + ------ + valid: bolean + access ok + t: numpy-array + numpy-array of time + signal: numpy-array + numpy-array of requested signals To get NaN, Infinity as their counter parts in Python (not 'Null') add the string +'&allowNaN=true' in the request. @@ -3670,7 +3775,7 @@ def read_restdb(request_url): #print('Error code: ', e.code) return False, [0], [-1] except Exception as E: - print('read_restdb: Error! ',E) + print('read_restdb: Error! ', E) return False, [0], [-1] else: # res = urllib.request.urlopen(request_url) @@ -3682,7 +3787,7 @@ def read_restdb(request_url): signal0=np.array(signal_list['values']) t=np.array(signal_list['dimensions']) except ValueError as e: - print('read_restdb: Error! ',signal_list['status'],e) + print('read_restdb: Error! ', signal_list['status'], e) return False, [0], [-2] else: return True, np.squeeze(t), np.squeeze(np.double(signal0)) @@ -3691,15 +3796,23 @@ def get_program_from_PID(instring): """ Retrieve program information via json-request (Python version of A.Alonso's Matlab routine getProgramInfo.m) - instring: specifies day and program ID (format yyyymmdd.pid) + input + ------ + instring : str + specifies day and program ID (format yyyymmdd.pid) + Returns - valid: retrival ok - program_list: for program ID req_PID - (returns whole list, if req_PID was not found) + ------ + valid: boolean + retrival ok + program_list: list + for program ID req_PID + (returns whole list, if req_PID was not found) Example: + ------ from W7Xrest import read_restdb as read_restdb - v,pl=read_restdb.get_program_from_PID('20160310.007') + v, pl=read_restdb.get_program_from_PID('20160310.007') if v: print(pl['description']) trigger6=pl['trigger']['6'][0] @@ -3709,12 +3822,12 @@ def get_program_from_PID(instring): string_date,req_PID=instring.split('.') except ValueError as e: print('get_program_from_PID: Format of input string should be similar to 20160310.007') - return False,0 + return False, 0 try: day=datetime.datetime.strptime(string_date, "%Y%m%d") except ValueError as e: print('get_program_from_PID: Date format should be similar to 20160310') - return False,0 + return False, 0 day_plus1=day+datetime.timedelta(days=1) # create timestamps from begining to end of that day @@ -3729,11 +3842,11 @@ def get_program_from_PID(instring): prog_list = json.loads(res.read().decode('utf-8')) res.close() except urllib.error.URLError as e: - print('get_program_from_PID: Error opening URL',e) - return False,0 + print('get_program_from_PID: Error opening URL', e) + return False, 0 except Exception as E: - print('get_program_from_PID: ERROR appeared',E) - return False,0 + print('get_program_from_PID: ERROR appeared', E) + return False, 0 else: pl=prog_list['programs'] id_list=[] @@ -3741,19 +3854,19 @@ def get_program_from_PID(instring): x=pr['id'] id_list.append(int((x.split('.'))[1])) try: - id=id_list.index(int(req_PID)) + id_=id_list.index(int(req_PID)) except ValueError as e: - print('get_program_from_PID: Could not find requested program ID',e) - return False,pl + print('get_program_from_PID: Could not find requested program ID', e) + return False, pl else: - return True, pl[id] + return True, pl[id_] """ end of the copied functions @author: thomsen """ -def get_trigger_from_PID(program, port, testmode=False, verbose=0): +def get_trigger_from_PID(program, port=None, testmode=False, verbose=0): '''Gets the nanosecond timestamps of beginning of preparation phase (t0), the start of experiment phase (t1) and the end of the experiment phase (t6). If there was no program in the Archive (calibration or test shot), the @@ -3784,49 +3897,12 @@ def get_trigger_from_PID(program, port, testmode=False, verbose=0): t_end: int nanosecond time stamp of end of experiment phase (t6) ''' - # handle different formats of port - if isinstance(port, str): - try: - # case port=='31' - port = int(port) - port = "AEF"+str(int(port)) - except: - # case port=='... AEF31 ...' - avports=[] - for key in portpathdict: - avports=avports+list(portpathdict[key].keys()) - portprefix=[] - for portstr in avports: - if portstr[0:3] not in portprefix: - portprefix.append(portstr[0:3]) - if len(port)<6: - if port not in avports: - print("get_trigger_from_PID: unknown port!",port,avports) - return False,0,0,0 - else: - gotit=False - for key in portprefix: - if key in port: - port = int(port[port.find(key)+3:port.find(key)+5]) - port=key+str(port) - gotit=True - break - if not gotit: - print("get_trigger_from_PID: unknown port!",port,avports) - return False,0,0,0 -# if 'AEF' in port: -# port = int(port[port.find('AEF')+3:port.find('AEF')+5]) - else: - try: - port="AEF"+str(int(port)) - except: - print("get_trigger_from_PID: port format not supported") - return False,0,0,0 + # check if program was defined in archive if not isinstance(program, str): - program=str(program) - if len(program.split(".")[0])<3: - program=program+"0" + program = str(program) + if len(program.split(".")[0]) < 3: + program = program+"0" prog = get_program_from_PID(program) if prog[0]: t_preparation = prog[1]['trigger']['0'][0] @@ -3835,7 +3911,45 @@ def get_trigger_from_PID(program, port, testmode=False, verbose=0): return True, t_preparation, t_start, t_end else: - if verbose>0: + # handle different formats of port + if isinstance(port, str): + try: + # case port=='31' + port = int(port) + port = "AEF"+str(int(port)) + except: + # case port=='... AEF31 ...' + avports = [] + for key in portpathdict: + avports = avports+list(portpathdict[key].keys()) + portprefix = [] + for portstr in avports: + if portstr[0:3] not in portprefix: + portprefix.append(portstr[0:3]) + if len(port)<6: + if port not in avports: + print("get_trigger_from_PID: unknown port!", port,avports) + return False, 0, 0, 0 + else: + gotit = False + for key in portprefix: + if key in port: + port = int(port[port.find(key)+3:port.find(key)+5]) + port = key+str(port) + gotit = True + break + if not gotit: + print("get_trigger_from_PID: unknown port!", port,avports) + return False, 0, 0, 0 + # if 'AEF' in port: + # port = int(port[port.find('AEF')+3:port.find('AEF')+5]) + else: + try: + port = "AEF"+str(int(port)) + except: + print("get_trigger_from_PID: port format not supported") + return False, 0, 0, 0 + if verbose > 0: print('get_trigger_from_PID: no program found in Archive. checking datastream...') date_str = program.split('.')[0] prog_no = int(program.split('.')[1]) @@ -3852,13 +3966,13 @@ def get_trigger_from_PID(program, port, testmode=False, verbose=0): signal_name = stream_name+'/'+version+'/0/raw' else: print("get_trigger_from_PID: no version with data found!") - return False,0,0,0 + return False, 0, 0, 0 # load all time intervals with data in this datastream intervals = AKF_2.get_time_intervals(signal_name, time_from, time_to) - frame_start = np.array(intervals[::-1,0]) + frame_start = np.array(intervals[::-1, 0]) frame_end = np.array(intervals[::-1,1]) # identify segments of frames by gaps in time steps - dt = abs(intervals[:-1,0]-intervals[1:,0]) + dt = abs(intervals[:-1, 0]-intervals[1:, 0]) dt_normal = np.median(dt) i_segment = np.hstack([np.array([0]), np.where(dt>2.5*dt_normal)[0]+1, np.array([len(intervals)])]) if verbose>1: @@ -3876,7 +3990,7 @@ def get_trigger_from_PID(program, port, testmode=False, verbose=0): frame_end[i_segment[(i+1)*3]-1] ] ) if verbose>1: print('get_trigger_from_PID: found {0} shots (NUC+BG+data)'.format(len(t_programs))) - if len(t_programs)>=prog_no and prog_no>0: + if len(t_programs) >= prog_no and prog_no>0: t_preparation = t_programs[prog_no-1][0] t_start = t_programs[prog_no-1][1] t_end = t_programs[prog_no-1][2] @@ -3900,15 +4014,15 @@ if __name__=='__main__': #%% temperature download and plotting example # port=31#"AEF50"#"AEF51"'20171114.053'# # prog="20180911.008"#"20181011.033"#'20171122.035'#'20181011.010'#'20180823.037'#'20170927.020'#"20181011.036"#"20181016.037"#"20180920.042"#"20171109.021"#"20181010.036" -# status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_window=[5,5.1],emi=0.80,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5) -# status2,time2,images2=download_raw_images_by_program(port,prog,time_window=0.02,verbose=5) -# bla=get_calib_data(50,program=prog,verbose=5) -# success,t,s,profile=extract_temperature_profile_from_DL(port,np.asarray(time-time[0])/1e9,images,profile="TM3h_5_5",verbose=10,reference_time=time[0]) +# status, time, images, valid=get_temp_from_raw_by_program(port, prog, time_window=[5,5.1], emi=0.80, t_version=2, version=0, threads=1, give_ERROR=False,use_firstframe_as_background=False, verbose=5) +# status2, time2, images2=download_raw_images_by_program(port, prog, time_window=0.02, verbose=5) +# bla=get_calib_data(50, program=prog, verbose=5) +# success, t, s, profile=extract_temperature_profile_from_DL(port, np.asarray(time-time[0])/1e9, images, profile="TM3h_5_5", verbose=10,reference_time=time[0]) # if status: # plt.figure() -# plt.imshow(images[-1],vmin=330,vmax=1000,cmap=exJet) +# plt.imshow(images[-1], vmin=330, vmax=1000, cmap=exJet) # cb=plt.colorbar() -# cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20) +# cb.set_label("temperature in K",rotation=270, labelpad=20, fontsize=20) # cb.ax.tick_params(labelsize=20) # plt.title("AEF{0}".format(port)) @@ -3921,7 +4035,7 @@ if __name__=='__main__': # plt.figure() # plt.subplot(2,1,1) # t_start = 2.0 -# status,times1,images1=download_heatflux_by_program(port,prog,time_window=[t_start,t_start+0.1],threads=4,version=2,verbose=3,givealpha=False,request=False) +# status, times1, images1=download_heatflux_by_program(port, prog, time_window=[t_start, t_start+0.1], threads=4, version=2, verbose=3, givealpha=False,request=False) # plot_frame1 = np.nan_to_num(images1[0])/1E6 # plot_frame1[plot_frame1<0.1] = 0 # q_max = np.nanmax(plot_frame1) @@ -3933,7 +4047,7 @@ if __name__=='__main__': # # plt.subplot(2,1,2) # t_start = 10.0 -# status,times2,images2=download_heatflux_by_program(port,prog,time_window=[t_start,t_start+0.1],threads=4,version=2,verbose=3,givealpha=False,request=False) +# status, times2, images2=download_heatflux_by_program(port, prog, time_window=[t_start, t_start+0.1], threads=4, version=2, verbose=3, givealpha=False,request=False) # plot_frame2 = np.nan_to_num(images2[0])/1E6 # plot_frame2[plot_frame2<0.1] = 0 # plt.scatter(mapping['X'], mapping['Y'], s=1, c=plot_frame2, cmap='jet', vmin=q_min, vmax=q_max) @@ -3948,7 +4062,7 @@ if __name__=='__main__': # prog = "20171109.008" # finger_no = 1605 # -# status, profile_time, s, profiles=get_heatflux_profile(port,finger_no,timepoint=1,program=prog,verbose=4) +# status, profile_time, s, profiles=get_heatflux_profile(port, finger_no, timepoint=1, program=prog, verbose=4) # plt.figure() # plt.plot(s, profiles[0]/1E6) # plt.xlabel('s [m]') @@ -3963,7 +4077,7 @@ if __name__=='__main__': # labels = [] # for port in [11,20,21,30,31,40,41]: # print('loading loads of port {0} in {1}'.format(port, prog)) -# exist,time,load,error=download_divertor_load(port,targetmodule=['all'],program=prog,request=False,verbose=5) +# exist, time, load, error=download_divertor_load(port, targetmodule=['all'], program=prog,request=False, verbose=5) # if exist: # load_start = np.mean(load[0][:10]) # load_end = np.mean(load[0][-10:]) @@ -3990,7 +4104,7 @@ if __name__=='__main__': # if exist: # plt.figure() # for i in range(len(keys)): -# plt.subplot(3,3,i+1) +# plt.subplot(3,3, i+1) # plt.imshow(model[keys[i]]) # plt.colorbar() # plt.title(keys[i]) @@ -3999,8 +4113,8 @@ if __name__=='__main__': #%% HDF5 writing test # import h5py as h5 -## bla=get_temp_from_raw_by_program_fullthreads(51,prog,time_window=[0,4],threads=4) -## bla2=get_nuced_raw_by_program_fullthreads(51,prog,time_window=[0,4],threads=4) +## bla=get_temp_from_raw_by_program_fullthreads(51, prog, time_window=[0,4], threads=4) +## bla2=get_nuced_raw_by_program_fullthreads(51, prog, time_window=[0,4], threads=4) # port = 10 # program = "20180925.013" # prog = get_program_from_PID(program) @@ -4009,34 +4123,34 @@ if __name__=='__main__': # t1 = prog[1]['trigger']['1'][0] # t6 = prog[1]['trigger']['6'][0] # -## bla=download_raw_images_by_times(port,t0,t1,verbose=5) -# bla = get_temp_from_raw_by_program(port,program,time_window=[0,4],emi=0.82, -# T_version=1,version=0,threads=4,give_ERROR=False, -# use_firstframe_as_background=True,verbose=5) +## bla=download_raw_images_by_times(port, t0, t1, verbose=5) +# bla = get_temp_from_raw_by_program(port, program, time_window=[0,4], emi=0.82, +# T_version=1, version=0, threads=4, give_ERROR=False, +# use_firstframe_as_background=True, verbose=5) # # filename = "AEF"+str(port)+"_"+program+"_temp_v1_b.h5" -# File = h5.File(filename,'w') +# File = h5.File(filename, 'w') # frames = np.asarray(bla[2]).swapaxes(0,2).swapaxes(0,1) # n_rows, n_cols, n_frames = np.shape(frames) -# dset = File.create_dataset('images',shape=(n_rows,n_cols,n_frames),dtype='uint16',chunks=(n_rows,n_cols,1)) +# dset = File.create_dataset('images', shape=(n_rows, n_cols, n_frames),dtype='uint16', chunks=(n_rows, n_cols,1)) # imags = np.ascontiguousarray(frames) # dset.write_direct(imags) -# File.create_dataset('timestamps', data=list(bla[1]), dtype='uint64')#,compression="gzip") +# File.create_dataset('timestamps', data=list(bla[1]), dtype='uint64')#, compression="gzip") # File.close() # # # test reading -# File = h5.File(filename,'r') +# File = h5.File(filename, 'r') # times = np.array(File['timestamps']) # frames2 = File['images'] # n_frames2 = frames2.shape[2] # if n_frames == n_frames2: -# i_frames = np.random.randint(low=0, high=n_frames-1 ,size=3) +# i_frames = np.random.randint(low=0, high=n_frames-1 , size=3) # for i in range(3): # print('testing frame', i_frames[i]) -# if (np.nan_to_num(frames[:,:,i_frames[i]]) == frames2[:,:,i_frames[i]]).all(): +# if (np.nan_to_num(frames[:,:, i_frames[i]]) == frames2[:,:, i_frames[i]]).all(): # print(' original and h5 file verions identical!') # else: -# print(' total difference of original and h5 file:', np.sum(np.nan_to_num(frames[:,:,i_frames[i]]) - frames2[:,:,i_frames[i]])) +# print(' total difference of original and h5 file:', np.sum(np.nan_to_num(frames[:,:, i_frames[i]]) - frames2[:,:, i_frames[i]])) # File.close() #%% get coldframe from NUC test