downloadversionIRdata.py

# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018
last update on Fr Nov 23 15:37:00 2018

Version: 3.0.2
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
@author: holn
"""

import numpy as np
import IR_image_tools as IR_tools
from IR_config_constants import archivepath,portcamdict,camlimdict,valid_FOV_circle,valid_background_rectangle,TC_port,TC_channel,IRCamColdframes_fittingpath,exJet,portpathdict,project,project_ana,heatflux_requestlist_path#,exJet_trans
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
import datetime
import urllib
import json
from PIL import Image
from io import BytesIO
from os.path import join
import matplotlib.pyplot as plt
try:          
    import threading
    import archivedb as AKF_2
    fastDL=True
except Exception as E:
    print(E)
    fastDL=False


try:
    config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
    FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0]   
    del FOV
except:
    config_path=""

def get_OP_by_program(program):
    try:    
        bla=int(program.split(".")[0])
        time_ns=int(TimeToNs([bla // 10000,bla % 10000 // 100,bla % 10000 % 100],[10,0,0,0]))
        return get_OP_by_time(time_ns)
    except Exception as E:
        print(E)
        return None
    
def get_OP_by_time(time_ns):
    dateOP=datetime.datetime.utcfromtimestamp(time_ns/1e9)
    if dateOP.year==2017:
        if dateOP.month>8 and dateOP.month<12:
            return "OP1.2a"
        elif dateOP.month==8 and dateOP.day>=28:
            return "OP1.2a"
        elif dateOP.month==12 and dateOP.day<8:
            return "OP1.2a"
        else:
            return None        
    elif dateOP.year==2018:
        return "OP1.2b"
    elif dateOP.year<=2016 and dateOP.year>=2015:
        if (dateOP.year==2016 and dateOP.month<=3) or (dateOP.year==2015 and dateOP.month==12):
            return "OP1.1"
        else:
            return None

def get_latest_version(stream,project="W7X",Test=False):
     """Find out latest version of given stream
     :param stream   url stream of interest
     :return int of version number of None if non-versioned or non-existing stream
     author: G. Schlisio, edit by holn
     """
     if Test:
         base="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
     else:
         base=archivepath    
     request =urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})
     try:
         response = urllib.request.urlopen(request)
         d = json.loads(response.read().decode('utf-8'))
     except urllib.error.HTTPError as ex:
         msg = ex.read()
         raise RuntimeError(msg)
     # detect unversioned or non-existing stream
     if d["versionInfo"] == []:
         return None
     versions = []
     for i in d['versionInfo']:
         versions.append(i['number'])

     return max(versions)

def TimeToNs(date,time):
    """
    TimeToNs(date,time)
    (list)  date    [year,month,day]
    (list)  time    [hours,minutes,seconds,microseconds]
    """ 
    date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3])
    div=date_time-datetime.datetime(1970,1,1,0,0,0)
    nsdate=div.total_seconds()*1e9
    return nsdate

def read_program(timestampstart,timestamp_end=0,tol=60):
    """
    read_program()
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
    if timestamp_end!=0:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestamp_end+tol*1e9))
    else:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestampstart+tol*1e9))
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw=res.read()
        res.close()
    except urllib.error.URLError as e:
        print('!get_program: Error opening URL')
        print(e)
        return False,0
    else:
        prog_string=prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl=prog_list['programs'][0]
        return True, pl

def read_restdb_old(request_url):
    """
    Reads JSON data from W7-X REST API
    Returns: 
        valid: access ok
        t: numpy-array of time
        signal: numpy-array of requested signals
    by H. Thomsen
    """
    try:
        res = urllib.request.urlopen(request_url)
    except urllib.error.URLError as e:
        print(e)
        return False, 0, -1
    else:
        signal_raw=res.read()
        res.close()
        signal_string=signal_raw.decode(encoding='UTF-8')
        signal_list = json.loads(signal_string)
        signal0=np.array(signal_list['values'])
        t=np.array(signal_list['dimensions'])
        return True, t, signal0

def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
    """
    download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
        time in ns
    Have to swap 11, 21 until correction in the database
    """    
    OP=IR_tools.get_OP_by_time(time_ns=time)
    if port == 21:
        port = 11
    elif port == 11:
        port = 21
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
    if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
        print("camera unknown, stopping here")
        raise Exception
    if version==0:
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM")
    if verbose>0:
        print("LUT V"+str(version)+" is used")
    #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
    LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
    if LUTpar[0]:
        LUTid=LUTpar[2][0]['structure'][query]
        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
        if LUTs[0]:
            LUTs=LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
            LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True,LUT
        else:
            print("Warning: unable to download the LUTs")
            del LUTpar, LUTs
            return False,0
    else:
        del LUTpar
        print("Warning: unable to find LUTs, check your request")
        return False,0


def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
    
    Read LUT V3 from local files for testing new calibration.
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
    OP = get_OP_by_time(this_time)
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
        with open(LUT_dir+"\\"+filename) as data_file:  
            jfile=json.load(data_file)
        LUT=np.array(jfile['LUT']).swapaxes(0,1)
        if verbose>0:
            print("succesfully loaded V3 LUT from local directory")
        return True, LUT
    except Exception as E:
        print("ERROR in loading V3 LUTs",E)
        return False, []
    
def download_NUC_by_program(port,program,exposure,version=0):
    prog=get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_NUC_by_times(port,starttime,stoptime,exposure,version)
    else:
#    except:
        print("cannot find the program")
        return False,0,0

def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
    project="W7X"
    OP=get_OP_by_time(starttime)
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM")
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        
        n=0
        nuctimes=[0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp']==exposure:
                nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i=NUCpar['structure']['offset_index']                
            n+=1 
        if nuctimes[0]!=0:
            NUC=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
            if NUC[0]:
                images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)                    
                return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
            else:
                print("NUC image for requested exposure time not found")
                return False, 0, 0
        else:
            print("NUC image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0

def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
    prog=get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose-1)
    else:
#    except:
        print("cannot find the program")
        return False,0,0
    
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if OP=="OP1.2a":
        t1=endtime
        t0=starttime
        prog=read_program(t1)
        program=prog[1]['id']
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):
            
            print("rebuilding coldframe")
            #use any way the rebuilt coldframe.
            sT = get_sensor_temp_by_program(port, program)[2][0]
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("unable to download reference frames")
            filestring = 'AEF' + str(port) + '_et' + str(t_exp)
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
            gain_error=0
            offset_error=0        
            return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        elif (port == 11) or (port == 21):
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("unable to download reference frames")
            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
            if NUC_DL[0]==False:
                raise Exception("NUC was not found")
            else:
                ci = NUC_DL[1][2]
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1)
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)                                
            gain_error=0
            offset_error=0
            return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        else:
            return download_NUC_by_times(port,starttime,endtime,t_exp)
    elif OP=="OP1.2b":
        gain_error=0
        offset_error=0
        gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
        if gotit:
            coldref=hotcold[1]
            hotref=hotcold[0]
        else:
            raise Exception("unable to download reference frames")
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
        exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
        if exist:
            cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
            for ele in coldframes:
                cold+=ele                
            cold=np.asarray(cold/len(coldframes),dtype=np.uint16)        
        else:
            return False,[0],[0]
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
        badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
        return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
    else:
        raise Exception("unknown Operation phase or NUC method not implemented for this OP")

def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0):
    """
    download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
    """
    gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(endtime-20e6))
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if gotit:
        expinds=np.where(texp_t==t_exp)[0]
        if len(expinds)==0:
            print("cannot find the exposure time in the given data")
            return False,[0],[0]                               
    else:
        print("exposure time not found")
        return False,[0],[0]
    gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(endtime-20e6))
    if gotitf:
        typiinds=np.where(values_f[expinds]==frametype)[0]
        if len(typiinds)>0:
            ref_t=[np.min(timef[expinds][typiinds]),np.max(timef[expinds][typiinds])]
#            print((ref_t[1]-ref_t[0])/1e9)
#            print(len(timef),len(timef[expinds]),len(timef[expinds][typiinds]))
        else:#okay the early data stuff or strange stuff
            print("frame type was not identified assuming that the first part is the cold and the second one the background")
            frametimes=time_t[expinds]
            diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
            turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
            if frametype==0:
                ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
                print((ref_t[1]-ref_t[0])/1e9)
            elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
                ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
                print((ref_t[1]-ref_t[0])/1e9)
            else:
                raise Exception("requested Frametype unknown and not implemented!")
    else:
        print("frametype not found")
        return False,[0],[0]
    t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
    t1date=t1date.isoformat()
    t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
    t0date=t0date.isoformat()
    if version==0:
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
    timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
    t_offset=timest0[-1][0]-time_t[0]
    if t_offset>10e6:
        print("time offset detected, try to correct this, offset is",t_offset)
    else:
        t_offset=0
#            print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
    return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset)
        
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        
        COLDtime=0
        HOTtime=0
        try:
            COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
        except:
            print("cold image for requested exposure time not found")
        try:
            HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
        except:
            print("hot image for requested exposure time not found")
        images=[]
        if HOTtime!=0:
            HOT=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
            if HOT[0]:
                images.append(HOT[2])
        if COLDtime!=0:
            COLD=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
            if COLD[0]:
                images.append(COLD[2])
                
        if HOT[0] and COLD[0] and len(images)==2:
            return True,images,['hot','cold']
        else:
            print("hot and cold image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
    
def download_background_by_program(port,program,exposure,version=0):
    '''
    returned exist,time,frame
    '''
    prog=get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_background_by_times(port,starttime,stoptime,exposure,version)
    else:#except:
        print("cannot find the program")
        return False,0,0
    
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
    stream=portpathdict[OP]["AEF"+str(port)]+"background_"
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
        n=0
        backtimes=[0]
        for backpar in signal_list['values']:
            if backpar['structure']['Texp']==exposure:
                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
                    if backpar['structure']['filter']==camera_filter:
                        backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
                else:
                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
            n+=1
        if backtimes[0]!=0:
            backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
            if backdat[0]:                                    
                return backdat#[True,backdat[2]]
            else:
                print("background image for requested exposure time(, filter) not found")
                return False,0,0
        else:
            print("background image for requested exposure time(, filter) not found")
            return False,0,0
    else:
        return False,0,0

def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False):
    "OP1.2b function"
    exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version)
    camera=portpathdict["OP1.2b"]["AEF"+str(port)]    
    background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
    for frame in backgroundframes:
        background+=frame
    background=np.asarray(background/len(backgroundframes),dtype=np.uint16)
    if plot_it:
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image unnuced")
    if not camera.split("_")[0]=="Infratec":
        background=apply_NUC([background],gain,offset)[0]
    if plot_it:        
        
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image nuced")
    return exist,btime[0],background

def download_raw_images_by_program(port,program,time_s=0,version=0,verbose=0):
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
        if type(time_s)==list or type(time_s)==np.ndarray:
            if len(time_s)>2 or time_s[0]>1000:
                raise Exception("invalid input for the timewindow!")
            tstart=time_s[0]
            tstop=time_s[1]
            if tstop<tstart:
                raise Exception("endtime before starttime")
            return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,verbose=verbose-1)
        else:
            if time_s==0:
                return download_raw_images_by_times(port,starttime,stoptime,version,verbose=verbose-1)
            else:
                return download_raw_images_by_times(port,starttime,int(starttime+time_s*1e9),version,verbose=verbose-1)
    except:
        if verbose>0:
            print("cannot find the program")
        return False,0,0

def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
    OP=IR_tools.get_OP_by_time(time_ns=starttime)    
    if version==0:
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"  
    return download_images_by_times(larchivepath,starttime=starttime,stoptime=stoptime,version=version,intervalSize=intervalSize,verbose=verbose-1)
    
def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
    if '_raw' in larchivepath:
        typo=np.uint16          
    else:
        typo=np.float32
    if (stoptime-starttime)/intervalSize>1:        
        nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
        if verbose>0:
            print("timewindow to large, splitting into smaller fractions ("+str(nrinterv)+")",(stoptime-starttime)/1e9)
        intervalls=[]
        for i in range(nrinterv):
            intervalls.append(int(starttime-10+i*intervalSize))
        intervalls.append(stoptime)
        time=-1
        allimages=0
        success=False
        for i in range(nrinterv):
            try:
                res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1]))
                signal_list = json.loads(res.read().decode('utf-8'))
                res.close()                
                images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
#                signal=np.array(signal_list['values'],dtype=np.uint16)
                t=np.array(signal_list['dimensions'])
                del signal_list
                if i==0:
                    allimages=images.copy()
                    time=t
                else:
                    allimages=allimages+images#np.append(ressignal,signal,axis=0)
                    time=np.append(time,t)
                del images,t
                success=True
            except urllib.error.URLError as e:
                
                if verbose>0:
                    print(e)                
        return success,time,allimages
    else:
        try:
            res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
            images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
#            signal=np.array(signal_list['values'],dtype=np.uint16)
            t=np.array(signal_list['dimensions'])
            del signal_list
            return True, t, images
        except urllib.error.URLError as e:
            if verbose>0:
                print(e)
            return False, 0,-1

if fastDL:
    def download_raw_images_by_program_via_png(port,program,time_s=0,version=0,threads=1, verbose=0):
        prog=get_program_from_PID(program)
#        try:
#            t_program = AKF_2.get_program_from_to(program)
#            prog =AKF_2.get_program_list(t_program[0], t_program[1])
        if not prog[0]:
            if verbose>0:
                print("program not found, cannot download the data")
                return False,[0],[0]
        else:
            starttime=prog[1]['trigger']['1'][0]
            stoptime=prog[1]['trigger']['6'][0]
            OP=get_OP_by_time(starttime)
            if version==0:
                version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
            larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
            return download_images_by_time_via_png(larchivepath,starttime,stoptime,time_s=time_s,threads=threads,version=version,verbose=verbose-1)
            
    def download_images_by_time_via_png(larchivepath,starttime,stoptime,time_s=0,threads=1,version=0,verbose=0):            
            success=True            
            if type(time_s)==list:
                tstart=time_s[0]
                tstop=time_s[1]
                if tstop<tstart:
                    raise Exception("endtime before starttime")
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
                stdate=stdate.isoformat()
                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
                enddate=enddate.isoformat()
            else:
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
                stdate=stdate.isoformat()
                if time_s==0:
                    enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
                    enddate=enddate.isoformat()
                else:
                    enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)  
                    enddate=enddate.isoformat()
                #"2017-11-15 08:00:00"
            try:
                times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
            except Exception as E:
                print(larchivepath)
                raise Warning(E)
                return False,[0],[0]            
            time=[]
            images=[]
            lnt=len(times)
            if threads==1:
                for i in range(lnt):
                    ele=times[lnt-1-i]
                    imag=download_last_raw_image_by_time(larchivepath,ele[0]-10,ele[0]+10)
                    if imag[0]:
                        time.append(ele[0])
                        images.append(np.array(imag[1],dtype=np.uint16))
                    else:
                        success=False
                return success,np.array(time),images
            else:
                tim=[]
                for i in range(lnt):
                    tim.append(times[lnt-1-i][0])
                intervalls=[]
                intervalSize=int(lnt/threads)
                for i in range(threads):
                    intervalls.append(int(i*intervalSize))
                intervalls.append(lnt)
                jobs = []
                resultdict = []
                for i in range(threads):
                    if verbose>0:
                        print("Start Thread ",i+1)                
                    TH=download_images_thread(i,larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,version)
                    jobs.append(TH)
                    TH.start()
                for p in jobs:
                    p.join()
                if verbose>0:
                    print("all threads are done")
                order=[]
                for ele in resultdict:
                    order.append(ele[0])
                    if len(np.where(np.asarray(ele[1])==False)[0])>0:
                        success=False                
                times=[]#np.array(resultdict[order.index(0)][2])
                images=[]#np.array(resultdict[order.index(0)][3])
                for i in range(threads):
                    images=images+resultdict[order.index(i)][3]
                    times=times+resultdict[order.index(i)][2]
                    resultdict[order.index(i)]=[]
                del resultdict
                return success,np.array(times),images
      
    class download_images_thread(threading.Thread):
        def __init__(self, threadID, larchivepath,times,resultlist,version=0):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            self.resultlist.append([self.threadID,successes,time,images])
    
    class convert_raw_to_temp_thread(threading.Thread):
        def __init__(self,larchivepath,times,resultlist,threadID,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
        def run(self):
            images=[]
            time=[]
            successes=[]        
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            if self.give_ERROR:
                sucess,images,error_images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR)
            else:
                success,images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR)
            
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
            print(datetime.datetime.now(),"correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels)
            self.resultlist.append([self.threadID,successes,time,images])
    
    class NUC_raw_thread(threading.Thread):
        def __init__ (self,larchivepath,times,resultlist,threadID,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            if self.give_ERROR:
                images,error_images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
            else:
                images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
            
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
            print(datetime.datetime.now(),"correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels)
            self.resultlist.append([self.threadID,successes,time,images]) 

def download_raw_FLIR_images_via_png(t1,t6,time_s=0,version=0,threads=1,verbose=0):    
    larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
    if type(time_s)==list:
        tstart=time_s[0]
        tstop=time_s[1]
        if tstop<tstart:
            raise Exception("endtime before starttime")
        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
        stdate=stdate.isoformat()
        enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)  
        enddate=enddate.isoformat()
    else:
        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
        stdate=stdate.isoformat()
        if time_s==0:
            enddate=datetime.datetime.utcfromtimestamp(t6/1e9)        
            enddate=enddate.isoformat()
        else:
            enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_s)  
            enddate=enddate.isoformat()
        #"2017-11-15 08:00:00"
    typ1_version=get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM")
    if typ1_version==None:
        mode=1
    else:
        mode=2
    if mode==1:
        ### okay the data is stored probably as different sets (so recorded with FLIR software)
        timess,versions=get_FLIR_Pset_times(stdate,enddate,version)
        if verbose>0:
            print("found ",np.shape(timess[0]),np.shape(timess[1]),np.shape(timess[2]),np.shape(timess[3])," frames in the different sets")
        ### we not know the times for all presets for the program, now we have to download them, if times is not empty
        exist,time,frames,tsets=download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=verbose-1)            
    elif mode==2:##the data is stored only in one datastream, at least this datastream exist
        raise Exception("not implemented")
        #download data and downlaod LUT
        
#        time=[]
#        images=[]
#        lnt=len(times)
#        if threads==1:
#            for i in range(lnt):
#                ele=times[lnt-1-i]
#                imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
#                if imag[0]:
#                    time.append(ele[0])
#                    images.append(np.array(imag[1],dtype=np.uint16))
#                else:
#                    success=False
#            return success,np.array(time),images
    else:
        raise Exception("cannot identifie the way the searched data is stored!")                        

    return exist,time,frames,mode,tsets,versions

def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0):
    larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
    if version==0:
        versions=[]
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS0_DATASTREAM"))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS1_DATASTREAM"))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS2_DATASTREAM"))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS3_DATASTREAM"))
    else:
        versions=[version,version,version,version]
    timess=[]
    if version==0:##no version specified search for data in all versions            
        counter=-1
        for version0 in versions:
            counter+=1
            if version0>0:
                found=False
                while (version0>0 and not found):                    
                    try:            
                        times_0=AKF_2.get_time_intervals(larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw",stdate.replace("T"," "),enddate.replace("T"," "))#
                        found=True
                    except Exception as E:
                        print(E,larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw")
                        found=False
                        times_0=[]
                        version0=version0-1
                timess.append(times_0)
            else:
                timess.append([])
    else:
        counter=-1
        for version0 in versions:
            counter+=1
            try:            
                times_0=AKF_2.get_time_intervals(larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw",stdate.replace("T"," "),enddate.replace("T"," "))#
            except Exception as E:
                print(E,larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw")
                times_0=[]
            timess.append(times_0)
    return timess,versions

def download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=0):
    if len(timess[0])==len(timess[1]) and len(timess[2])==len(timess[3]) and len(timess[0])==0:
            exist=False
            time=[]
            frames=[]
            tsets=[]
    else:#timess is not empty, there is data in at least one Pset
        if verbose>0:
            print(datetime.datetime.now(),"data found, start the download")
        exist=True
        timesets=[]
        framess=[]
        success=True
        Psets=[]
        if threads==1:            
            counter=-1
            setn=-1
            for times in timess:                    
                setn+=1
                if times!=[]:
                    counter+=1
                    Psets.append(counter)
                    framess.append([])
                    timesets.append([])
                    lnt=len(times)                    
                    for i in range(lnt):                        
                        ele=times[lnt-1-i]
                        imag=download_last_raw_image_by_time(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw",ele[0]-10,ele[0]+10)
                        if imag[0]:
                            timesets[counter].append([ele[0],i,setn,counter])
                            framess[counter].append(np.array(imag[1],dtype=np.uint16))
                        else:
                            success=False
        else:#several threads, each for a preset
            tim=[]
            counter=-1
            setn=-1
            threads=0
            larchivepaths=[]
            versionss=[]
            for times in timess:                    
                times=np.array(times)
                setn+=1
                if times!=[]:
                    counter+=1
                    threads+=1
                    Psets.append(counter)
                    larchivepaths.append(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw")
                    tim.append(times[:,0])
                    versionss.append(versions[setn])
            jobs = []
            resultdict = []
            for i in range(threads):
                if verbose>0:
                    print("Start Thread ",i+1)                
                TH=download_images_thread(i,larchivepaths[i],tim[i],resultdict,versionss[i])
                jobs.append(TH)
                TH.start()            
            for p in jobs:
                p.join()
            if verbose>0:
                print("all threads are done")
            order=[]
            for ele in resultdict:
                order.append(ele[0])
                if len(np.where(np.asarray(ele[1])==False)[0])>0:
                    success=False                
            for i in range(threads):
                framess.append(resultdict[order.index(i)][3])
                timeline=[]
                for j in range(len(resultdict[order.index(i)][2])):
                    timeline.append([resultdict[order.index(i)][2][j],j,Psets[i],i])
                timesets.append(timeline)
                resultdict[order.index(i)]=[]
            del resultdict,timeline
        if verbose>0:
            print(datetime.datetime.now(),"download finished, successfull",success)
        if success:
            del timess
            ### okay the have now the data from all presets, now we have to sort the frames into one array
            time=[]
            frames=[]                
            dummyT=[]
            tsets=[]
            for tim in timesets:
                dummyT=dummyT+tim
            dummyT.sort()
            for ele in dummyT: