downloadversionIRdata.py

# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018
last update on Fr Nov 23 15:37:00 2018

Version: 3.2.1
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
@author: holn
"""

import numpy as np
import IR_image_tools as IR_tools
from IR_config_constants import archivepath,portcamdict,camlimdict, \
                                valid_FOV_circle,valid_background_rectangle, \
                                TC_port,TC_channel,IRCamColdframes_fittingpath, \
                                exJet,portpathdict,project,project_ana, \
                                heatflux_requestlist_path#,exJet_trans
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
import datetime
import urllib
import json
from PIL import Image
from io import BytesIO
from os.path import join
import matplotlib.pyplot as plt
try:          
    import threading
    import archivedb as AKF_2
    fastDL=True
except Exception as E:
    print(E)
    fastDL=False


try:
    config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
    FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0]   
    del FOV
except:
    config_path=""


def get_latest_version(stream,project="W7X",Test=False,t_from=None,t_to=None,program=None):
     """Find out latest version of given stream
     :param stream   url stream of interest
     :return int of version number of None if non-versioned or non-existing stream
     author: G. Schlisio, edit by holn
     """
     if Test:
         base="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
     else:
         base=archivepath    

    
     if program==None and t_from==None and t_to==None:         
         request =urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})            
     else:
         if program is not None:
             prog=get_program_from_PID(program)
             if prog[0]:
                 t_from=prog[1]['trigger']['0'][0]
                 t_to=prog[1]['trigger']['6'][0]
         elif t_from is not None and t_to==None:
             t_to=int(t_from+100e9)
         elif t_from==None and t_to is not None:
             t_from=int(t_to-10e9)
         request =urllib.request.Request(base +project+"/" + stream + "/_versions.json?from="+str(t_from)+"&upto="+str(t_to), headers={"Accept": "application/json"})
     try:
         response = urllib.request.urlopen(request)
         d = json.loads(response.read().decode('utf-8'))
     except urllib.error.HTTPError as ex:
         msg = ex.read()
         raise RuntimeError(msg)
     # detect unversioned or non-existing stream
     if d["versionInfo"] == []:
         return None
     versions = []
     for i in d['versionInfo']:
         versions.append(i['number'])

     return max(versions)

def TimeToNs(date,time):
    """
    TimeToNs(date,time)
    (list)  date    [year,month,day]
    (list)  time    [hours,minutes,seconds,microseconds]
    """ 
    date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3])
    div=date_time-datetime.datetime(1970,1,1,0,0,0)
    nsdate=div.total_seconds()*1e9
    return nsdate

def read_program(timestamp_start,timestamp_end=0,tol=60):
    """
    read_program()
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
    if timestamp_end==0:
        timestamp_end = timestamp_start
    jsonurl = '{0}={1}&upto={2}'.format(program_url,
                                        int(timestamp_start-tol*1e9),
                                        int(timestamp_end+tol*1e9))
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw=res.read()
        res.close()
    except urllib.error.URLError as e:
        print('read_program: Error opening URL')
        print(e)
        return False,0
    else:
        prog_string=prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl=prog_list['programs'][0]
        return True, pl

def read_restdb_old(request_url):
    """
    Reads JSON data from W7-X REST API
    Returns: 
        valid: access ok
        t: numpy-array of time
        signal: numpy-array of requested signals
    by H. Thomsen
    """
    try:
        res = urllib.request.urlopen(request_url)
    except urllib.error.URLError as e:
        print('read_restdb_old: Error ', e)
        return False, 0, -1
    else:
        signal_raw=res.read()
        res.close()
        signal_string=signal_raw.decode(encoding='UTF-8')
        signal_list = json.loads(signal_string)
        signal0=np.array(signal_list['values'])
        t=np.array(signal_list['dimensions'])
        return True, t, signal0

def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
    """
    download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
        time in ns
    Have to swap 11, 21 until correction in the database
    """    
    OP=IR_tools.get_OP_by_time(time_ns=time)
    if port == 21:
        port = 11
    elif port == 11:
        port = 21
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
    if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
        print("download_LUT: Error! Camera unknown, stopping here.")
        raise Exception
    if version==0:
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM")
    if verbose>0:
        print("download_LUT: LUT V"+str(version)+" is used")
    #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
    LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
    if LUTpar[0]:
        LUTid=LUTpar[2][0]['structure'][query]
        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
        if LUTs[0]:
            LUTs=LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
            LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True,LUT
        else:
            print("download_LUT: Warning! unable to download the LUTs")
            del LUTpar, LUTs
            return False,0
    else:
        del LUTpar
        print("download_LUT: Warning! unable to find LUTs, check your request")
        return False,0


def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
    
    Read LUT V3 from local files for testing new calibration.
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
    OP = IR_tools.get_OP_by_time(time_ns=this_time)
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
        with open(LUT_dir+"\\"+filename) as data_file:  
            jfile=json.load(data_file)
        LUT=np.array(jfile['LUT']).swapaxes(0,1)
        if verbose>0:
            print("read_LUT_from_file: succesfully loaded V3 LUT from local directory")
        return True, LUT
    except Exception as E:
        print("read_LUT_from_file: ERROR in loading V3 LUTs",E)
        return False, []
    
def download_NUC_by_program(port,program,exposure,version=0):
    prog=get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_NUC_by_times(port,starttime,stoptime,exposure,version)
    else:
#    except:
        print("download_NUC_by_program: cannot find the program")
        return False,0,0

def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
    project="W7X"
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM")
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print('download_NUC_by_times: Error! ', e)
        goon=False
    if goon:        
        n=0
        nuctimes=[0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp']==exposure:
                nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i=NUCpar['structure']['offset_index']                
            n+=1 
        if nuctimes[0]!=0:
            NUC=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
            if NUC[0]:
                images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)                    
                return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
            else:
                print("download_NUC_by_times: NUC image for requested exposure time not found")
                return False, 0, 0
        else:
            print("download_NUC_by_times: NUC image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0

def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
    prog=get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose-1)
    else:
#    except:
        print("get_NUC_by_program: Error! cannot find the program")
        return False,0,0
    
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
    """
    Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera
    in a given port and time interval. Depending on time and camera, reconstruct
    teh cold frame and/or compute the gain,offset from the hot and cold frame.
    """
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if OP=="OP1.2a":
        t1=endtime
        t0=starttime
        prog=read_program(t1)
        program=prog[1]['id']
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):
            if verbose>0:
                print("get_NUC_by_times: rebuilding coldframe")
            #use any way the rebuilt coldframe.
            sT = get_sensor_temp_by_program(port, program)[2][0]
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("get_NUC_by_times: unable to download reference frames")
            filestring = 'AEF' + str(port) + '_et' + str(t_exp)
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
            if verbose>0:
                print('get_NUC_by_times: calculate gain, offset and bad pixels')
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)
            gain_error=0
            offset_error=0        
            return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        elif (port == 11) or (port == 21):
            if verbose>0:
                print('get_NUC_by_times: downloading NUC')
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("get_NUC_by_times: unable to download reference frames")
            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
            if NUC_DL[0]==False:
                raise Exception("get_NUC_by_times: NUC was not found")
            else:
                ci = NUC_DL[1][2]
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1)
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)                                
            gain_error=0
            offset_error=0
            return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        else:
            if verbose>0:
                print('get_NUC_by_times: downloading NUC')
            return download_NUC_by_times(port,starttime,endtime,t_exp)
    elif OP=="OP1.2b":
        gain_error=0
        offset_error=0
        gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
        if gotit:
            coldref=hotcold[1]
            hotref=hotcold[0]
        else:
            raise Exception("get_NUC_by_times: unable to download reference frames")
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
        exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0,verbose=verbose-1)
        if exist:
            cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
            for ele in coldframes:
                cold+=ele                
            cold=np.asarray(cold/len(coldframes),dtype=np.uint16)        
        else:
            return False,[0],[0]
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
        badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)
        return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
    else:
        raise Exception("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP")

def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0,verbose=0):
    """
    download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
    """
    gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(endtime-20e6))
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if gotit:
        expinds=np.where(texp_t==t_exp)[0]
        if len(expinds)==0:
            print("download_calibration_raw_files_by_time: Error! cannot find the exposure time in the given data")
            return False,[0],[0]                               
    else:
        print("download_calibration_raw_files_by_time: Error! exposure time not found")
        return False,[0],[0]
    gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(endtime-20e6))
    if gotitf:
        typiinds=np.where(values_f[expinds]==frametype)[0]
        if len(typiinds)>0:
            ref_t=[np.min(timef[expinds][typiinds]),np.max(timef[expinds][typiinds])]
#            print((ref_t[1]-ref_t[0])/1e9)
#            print(len(timef),len(timef[expinds]),len(timef[expinds][typiinds]))
        else:#okay the early data stuff or strange stuff
            if verbose>0:
                print("download_calibration_raw_files_by_time: frame type was not identified assuming that the first part is the cold and the second one the background")
            frametimes=time_t[expinds]
            diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
            turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
            if frametype==0:
                ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
                print((ref_t[1]-ref_t[0])/1e9)
            elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
                ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
                print((ref_t[1]-ref_t[0])/1e9)
            else:
                raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
    else:
        print("download_calibration_raw_files_by_time: Error! frametype not found")
        frametimes=time_t[expinds]
        diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
        turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
        if frametype==0:
            ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
            print((ref_t[1]-ref_t[0])/1e9)
        elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
            ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
            print((ref_t[1]-ref_t[0])/1e9)
        else:
            raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
#        return False,[0],[0]
    t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
    t1date=t1date.isoformat()
    t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
    t0date=t0date.isoformat()
    if version==0:
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=endtime)
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
    timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
    t_offset=timest0[-1][0]-time_t[0]
    if t_offset>10e6:
        if verbose>0:
            print("download_calibration_raw_files_by_time: time offset detected, try to correct this, offset is",t_offset)
    else:
        t_offset=0
#            print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
    return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset,verbose=verbose-1)
        
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
    """
    Loads the most recent hot and cold calibration frames for a starttime.
    Uses first calibration frames if time is not defined.
    """
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime)
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print('download_hot_cold_reference_by_times: Error! ',e)
        goon=False
    if goon:        
        COLDtime=0
        HOTtime=0
        try:
            COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
        except:
            print("download_hot_cold_reference_by_times: cold image for requested exposure time not found")
        try:
            HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
        except:
            print("download_hot_cold_reference_by_times: hot image for requested exposure time not found")
        images=[]
        if HOTtime!=0:
            HOT=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
            if HOT[0]:
                images.append(HOT[2])
        if COLDtime!=0:
            COLD=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
            if COLD[0]:
                images.append(COLD[2])
                
        if HOT[0] and COLD[0] and len(images)==2:
            return True,images,['hot','cold']
        else:
            print("download_hot_cold_reference_by_times: Error! hot and cold image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
    
def download_background_by_program(port,program,exposure,version=0):
    '''
    returned exist,time,frame
    '''
    prog=get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_background_by_times(port,starttime,stoptime,exposure,version)
    else:#except:
        print("download_background_by_program: Error! cannot find the program {0}".format(program))
        return False,0,0
    
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
    stream=portpathdict[OP]["AEF"+str(port)]+"background_"
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print('download_background_by_times: Error! ',e)
        goon=False
    if goon: 
        n=0
        backtimes=[0]
        for backpar in signal_list['values']:
            if backpar['structure']['Texp']==exposure:
                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
                    if backpar['structure']['filter']==camera_filter:
                        backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
                else:
                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
            n+=1
        if backtimes[0]!=0:
            backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
            if backdat[0]:                                    
                return backdat#[True,backdat[2]]
            else:
                print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
                return False,0,0
        else:
            print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
            return False,0,0
    else:
        return False,0,0

def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False,verbose=0):
    """
    OP1.2b function
    """
    exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version,verbose=verbose-1)
    camera=portcamdict["OP1.2b"]["AEF"+str(port)]  
    if verbose>50:
        print("camera is",camera)
    background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
    for frame in backgroundframes:
        background+=frame
    background=np.asarray(background/len(backgroundframes),dtype=np.uint16)
    if plot_it:
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image unnuced")
    if not camera.split("_")[0]=="Infratec":
        background=apply_NUC([background],gain,offset)[0]
    if plot_it:        
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image nuced")
    return exist,btime[0],background

def download_raw_images_by_program(port,program,time_window=0,version=0,verbose=0):
    """
    """
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
        if type(time_window)==list or type(time_window)==np.ndarray:
            if len(time_window)>2 or time_window[0]>1000:
                raise Exception("download_raw_images_by_program: invalid input for the time_window!")
            tstart=time_window[0]
            tstop=time_window[1]
            if tstop<tstart:
                raise Exception("download_raw_images_by_program: endtime before starttime")
            return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,verbose=verbose-1)
        else:
            if time_window==0:
                return download_raw_images_by_times(port,starttime,stoptime,version,verbose=verbose-1)
            else:
                return download_raw_images_by_times(port,starttime,int(starttime+time_window*1e9),version,verbose=verbose-1)
    except:
        print("download_raw_images_by_program: Error! cannot find the program")
        return False,0,0

def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
    """
    """
    OP=IR_tools.get_OP_by_time(time_ns=starttime)    
    if version==0:
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime)
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"  
    return download_images_by_times(larchivepath,starttime=starttime,stoptime=stoptime,version=version,intervalSize=intervalSize,verbose=verbose-1)
    
def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
    """
    """
    if '_raw' in larchivepath:
        typo=np.uint16          
    else:
        typo=np.float32
    if (stoptime-starttime)/intervalSize>1:        
        nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
        if verbose>0:
            print("download_images_by_times: time_window to large, splitting {0}s interval into {1} smaller fractions".format((stoptime-starttime)/1e9,nrinterv))
        intervalls=[]
        for i in range(nrinterv):
            intervalls.append(int(starttime-10+i*intervalSize))
        intervalls.append(stoptime)
        time=-1
        allimages=0
        success=False
        for i in range(nrinterv):
            try:
                res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1]))
                signal_list = json.loads(res.read().decode('utf-8'))
                res.close()                
                images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
#                signal=np.array(signal_list['values'],dtype=np.uint16)
                t=np.array(signal_list['dimensions'])
                del signal_list
                if i==0:
                    allimages=images.copy()
                    time=t
                else:
                    allimages=allimages+images#np.append(ressignal,signal,axis=0)
                    time=np.append(time,t)
                del images,t
                success=True
            except urllib.error.URLError as e:
                print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e))                
        return success,time,allimages
    else:
        try:
            if verbose>100:
                print(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))
            res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))            
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
            images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
#            signal=np.array(signal_list['values'],dtype=np.uint16)
            t=np.array(signal_list['dimensions'])
            del signal_list
            return True, t, images
        except urllib.error.URLError as e:
            print('download_images_by_times: Error! ',e)
            return False, 0,-1

if fastDL:
    def download_raw_images_by_program_via_png(port,program,time_window=0,version=0,threads=1,verbose=0):
        """
        """
        prog=get_program_from_PID(program)
#        try:
#            t_program = AKF_2.get_program_from_to(program)
#            prog =AKF_2.get_program_list(t_program[0], t_program[1])
        if not prog[0]:
            print("download_raw_images_by_program_via_png: Error! program not found, cannot download the data")
            return False,[0],[0]
        else:
            starttime=prog[1]['trigger']['1'][0]
            stoptime=prog[1]['trigger']['6'][0]
            OP=IR_tools.get_OP_by_time(time_ns=starttime)
            if version==0:
                version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime)
            larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
            return download_images_by_time_via_png(larchivepath,starttime,stoptime,
                                                   time_window=time_window,threads=threads,
                                                   version=version,verbose=verbose-1)
            
    def download_images_by_time_via_png(larchivepath,starttime,stoptime,time_window=0,
                                        threads=1,version=0,verbose=0,framerate='max'):
        """
        """
        success=True            
        if type(time_window)==list:
            tstart=time_window[0]
            tstop=time_window[1]
            if tstop<tstart:
                raise Exception("download_images_by_time_via_png: endtime before starttime")
            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
            stdate=stdate.isoformat()
            enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
            enddate=enddate.isoformat()
        else:
            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
            stdate=stdate.isoformat()
            if time_window==0:
                enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
                enddate=enddate.isoformat()
            else:
                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window)  
                enddate=enddate.isoformat()
            #"2017-11-15 08:00:00"
        try:
            times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
        except Exception as E:
            print('download_images_by_time_via_png: Error loading times from ',larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))
            raise Warning(E)
            return False,[0],[0]  
            
        if framerate!='max': 
            if verbose>5:
                print("download_images_by_time_via_png: downsampling active; number of original frames:",len(times))
            if type(framerate)==float or type(framerate)==int:
                realrate=1e9/((times[0]-times[-1])[0]/len(times))
                if framerate<realrate:
                    steps=int(round(realrate/framerate,0))
                    dummy=[]
                    for i in range(0,len(times),steps):
                        dummy.append(times[i])
                    times=dummy
            if verbose>5:
                print("download_images_by_time_via_png: downsampling active; number of downsampeld frames:",len(times),steps)
        time=[]
        images=[]
        lnt=len(times)
        if threads==1:
            for i in range(lnt):
                ele=times[lnt-1-i]
                imag=download_last_raw_image_by_time(larchivepath,ele[0]-10,ele[0]+10)
                if imag[0]:
                    time.append(ele[0])
                    images.append(np.array(imag[1],dtype=np.uint16))
                else:
                    success=False
            return success,np.array(time),images
        else:
            tim=[]
            for i in range(lnt):
                tim.append(times[lnt-1-i][0])
            intervalls=[]
            intervalSize=int(lnt/threads)
            for i in range(threads):
                intervalls.append(int(i*intervalSize))
            intervalls.append(lnt)
            jobs = []
            resultdict = []
            for i in range(threads):
                if verbose>0:
                    print("download_images_by_time_via_png: Start Thread ",i+1)                
                TH=download_images_thread(i,larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,version)
                jobs.append(TH)
                TH.start()
            for p in jobs:
                p.join()
            if verbose>0:
                print("download_images_by_time_via_png: all threads are done")
            order=[]
            for ele in resultdict:
                order.append(ele[0])
                if len(np.where(np.asarray(ele[1])==False)[0])>0:
                    success=False                
            times=[]#np.array(resultdict[order.index(0)][2])
            images=[]#np.array(resultdict[order.index(0)][3])
            for i in range(threads):
                images=images+resultdict[order.index(i)][3]
                times=times+resultdict[order.index(i)][2]
                resultdict[order.index(i)]=[]
            del resultdict
            return success,np.array(times),images
      
    class download_images_thread(threading.Thread):
        def __init__(self, threadID, larchivepath,times,resultlist,version=0):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            self.resultlist.append([self.threadID,successes,time,images])
    
    class convert_raw_to_temp_thread(threading.Thread):
        def __init__(self,larchivepath,times,resultlist,threadID,version=0,background=0,
                     LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,
                     give_ERROR=False,FOV=0,badpixels=[],verbose=0):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
            self.verbose=verbose
        def run(self):
            images=[]
            time=[]
            successes=[]        
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            if self.give_ERROR:
                sucess,images,error_images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR,verbose=self.verbose-1)
            else:
                success,images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR,verbose=self.verbose-1)
            
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
            if self.verbose>0:
                print(datetime.datetime.now(),"convert_raw_to_temp_thread: correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
            self.resultlist.append([self.threadID,successes,time,images])
    
    class NUC_raw_thread(threading.Thread):
        def __init__ (self,larchivepath,times,resultlist,threadID,version=0,background=0,
                      LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,
                      give_ERROR=False,FOV=0,badpixels=[],verbose=0):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
            self.verbose=verbose
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            if self.verbose>0:
                print(datetime.datetime.now(),"NUC_raw_thread: applying NUC")
            if self.give_ERROR:
                images,error_images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
            else:
                images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
                
            if self.verbose>0:
                print(datetime.datetime.now(),"NUC_raw_thread: correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
            self.resultlist.append([self.threadID,successes,time,images]) 

def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,verbose=0,Test=True):
    """
    """
    if Test:
        larchivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+"W7X/QSR07_FLIR/AEK51_raw"
    else:
        larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
    if type(time_window)==list:
        tstart=time_window[0]
        tstop=time_window[1]
        if tstop<tstart:
            raise Exception("download_raw_FLIR_images_via_png: endtime before starttime")
        starttime=int(t1-100+tstart*1e9)
        endtime=int(t1+tstop*1e9)
#        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
#        stdate=stdate.isoformat()
#        enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)  
#        enddate=enddate.isoformat()
    else:
        starttime=int(t1-100)
        if time_window==0:
            endtime=t6
        else:
            endtime=int((t1)+time_window*1e9)
#        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
#        stdate=stdate.isoformat()
#        if time_window==0:
#            enddate=datetime.datetime.utcfromtimestamp(t6/1e9)        
#            enddate=enddate.isoformat()
#        else:
#            enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_window)  
#            enddate=enddate.isoformat()
        #"2017-11-15 08:00:00"
    typ1_version=get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM",Test=Test,t_from=tstart)
    if typ1_version==None:
        mode=1
    else:
        mode=2
    if mode==1:
        ### okay the data is stored probably as different sets (so recorded with FLIR software)
        timess,versions=get_FLIR_Pset_times(starttime,endtime,version,Test=Test)#stdate,enddate,version,Test=Test)
        if verbose>0:
            print("download_raw_FLIR_images_via_png: found {0} frames in the different sets".format([np.shape(x) for x in timess]))
        ### we not know the times for all presets for the program, now we have to download them, if times is not empty
        exist,time,frames,tsets=download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=verbose-1)            
    elif mode==2:##the data is stored only in one datastream, at least this datastream exist
        raise Exception("download_raw_FLIR_images_via_png: not implemented")
        #download data and downlaod LUT
        
#        time=[]
#        images=[]
#        lnt=len(times)
#        if threads==1:
#            for i in range(lnt):
#                ele=times[lnt-1-i]
#                imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
#                if imag[0]:
#                    time.append(ele[0])
#                    images.append(np.array(imag[1],dtype=np.uint16))
#                else:
#                    success=False
#            return success,np.array(time),images
    else:
        raise Exception("download_raw_FLIR_images_via_png: cannot identifie the way the searched data is stored!")                        

    return exist,time,frames,mode,tsets,versions

def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0,Test=True):
    """
    """
    if Test:
        larchivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+"W7X/QSR07_FLIR/AEK51_raw"
    else:
        larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
    if version==0:
        versions=[]
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS0_DATASTREAM",Test=Test,t_from=stdate))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS1_DATASTREAM",Test=Test,t_from=stdate))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS2_DATASTREAM",Test=Test,t_from=stdate))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS3_DATASTREAM",Test=Test,t_from=stdate))
    else:
        versions=[version,version,version,version]
    timess=[]
    if version==0:
        if verbose>0:
            print('get_FLIR_Pset_times: no version specified search for data in all versions')