downloadversionIRdata.py

# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017

@author: holn
"""

import numpy as np
try:
    import W7Xrest.read_restdb as AKF_1
except:
    import read_restdb as AKF_1
import datetime
import urllib
import json
from PIL import Image
from io import BytesIO
import matplotlib.pyplot as plt
try:        
    import multiprocessing    
    try:
        import holn.archivedb as AKF_2
    except:
        import archivedb as AKF_2
    fastDL=True
except Exception as E:
    print(E)
    fastDL=False

archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/W7X/"

portcamdict={
    'OP1.2a':{
        'AEF10': 'IRCam_Caleo768kL_0901',
        'AEF11': 'IRCam_Caleo768kL_0906',
        'AEF20': 'IRCam_Caleo768kL_0702',
        'AEF21': 'IRCam_Caleo768kL_0904',
        'AEF30': 'IRCam_Caleo768kL_0907',
        'AEF31': 'IRCam_Caleo768kL_0903',
        'AEF40': 'IRCam_Caleo768kL_0701',
        'AEF41': 'IRCam_Caleo768kL_0902',
        'AEF50': 'Infratec_9312907',
        'AEF51': 'IRCam_Caleo768kL_0905'}
    	}

try:
    config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\v1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
    FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0]   
    del FOV
except:
    config_path=""



camlimdict={
    'IRCam_Caleo768kL_0901':
        {
        1:(50,5000),
        2:(50,3000),
        3:(50,2050),
        4:(50,1200),
        5:(50,1100),
        6:(50,850),
        7:(50,800),
        8:(50,600),
        9:(50,550)
         },
    'IRCam_Caleo768kL_0902':
        {
        1:(50,5000),
        2:(50,3000),
        3:(50,2050),
        4:(50,1200),
        5:(50,1100),
        6:(50,1000),
        7:(50,800),
        8:(50,800),
        9:(50,650)
         },
    'IRCam_Caleo768kL_0903':
        {
        1:(50,5000),
        2:(50,3000),
        3:(50,2050),
        4:(50,1200),
        5:(50,1100),
        6:(50,1000),
        7:(50,900),
        8:(50,800),
        9:(50,700)
         },
    'IRCam_Caleo768kL_0904':
        {
        1:(50,5000),
        2:(50,3000),
        3:(50,2050),
        4:(50,1200),
        5:(50,1100),
        6:(50,900),
        7:(50,800),
        8:(50,700),
        9:(50,600)
         },
    'IRCam_Caleo768kL_0905':
        {
        1:(50,5000),
        2:(50,3000),
        3:(50,2050),
        4:(50,1200),
        5:(50,1100),
        6:(50,900),
        7:(50,800),
        8:(50,700),
        9:(50,600)
         },
    'IRCam_Caleo768kL_0906':
        {
        1:(50,5000),
        2:(50,3000),
        3:(50,2050),
        4:(50,1200),
        5:(50,1000),
        6:(50,800),
        7:(50,750),
        8:(50,600),
        9:(50,550)
         },
    'IRCam_Caleo768kL_0907':
        {
        1:(50,5000),
        2:(50,3000),
        3:(50,2050),
        4:(50,1200),
        5:(50,1100),
        6:(50,90),
        7:(50,800),
        8:(50,700),
        9:(50,600)
         },
    'IRCam_Caleo768kL_0701':        
        {
        1:(50,5000),
        2:(50,3000),
        3:(50,2050),
        4:(50,1580),
        5:(50,1100),
        6:(50,900),
        7:(50,800),
        8:(50,750),
        9:(50,700)
         },
    'IRCam_Caleo768kL_0702':
        {
        1:(50,5000),
        2:(50,3000),
        3:(50,2050),
        4:(50,1580),
        5:(50,1150),
        6:(50,1000),
        7:(50,800),
        8:(50,750),
        9:(50,700)
         },
     'Infratec_9312907':
         {
             0:{
             25:(50,400),
             50:(50,300),
            100:(50,250),
            200:(50,200),
            400:(50,150),
            800:(50,100)
             },
             1:{
             25:(300,1200),
             50:(200,800),
            100:(150,650),
            200:(100,500),
            400:(50,400),
            800:(50,300)
             },
            2:{
             25:(200,1200),
             50:(200,1200),
            100:(200,1200),
            200:(200,1200),
            400:(200,1000),
            800:(200,700)
             },
         }
}

def get_OP_by_program(program):
    try:    
        bla=int(program.split(".")[0])
        time_ns=int(TimeToNs([bla // 10000,bla % 10000 // 100,bla % 10000 % 100],[10,0,0,0]))
        return get_OP_by_time(time_ns)
    except Exception as E:
        print(E)
        return None
    
def get_OP_by_time(time_ns):
    dateOP=datetime.datetime.utcfromtimestamp(time_ns/1e9)
    if dateOP.year==2017:
        if dateOP.month>8 and dateOP.month<12:
            return "OP1.2a"
        elif dateOP.month==8 and dateOP.day>28:
            return "OP1.2a"
        elif dateOP.month==12 and dateOP.day<8:
            return "OP1.2a"
        else:
            return None        
    elif dateOP.year==2018:
        return "OP1.2b"
    elif dateOP.year<=2016 and dateOP.year>=2015:
        if (dateOP.year==2016 and dateOP.month<=3) or (dateOP.year==2015 and dateOP.month==12):
            return "OP1.1"
        else:
            return None

def TimeToNs(date,time):
    """
    TimeToNs(date,time)
    (list)  date    [year,month,day]
    (list)  time    [hours,minutes,seconds,microseconds]
    """ 
    date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3])
    div=date_time-datetime.datetime(1970,1,1,0,0,0)
    nsdate=div.total_seconds()*1e9
    return nsdate

def read_program(timestampstart,timestamp_end=0,tol=60):
    """
    read_program()
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
    if timestamp_end!=0:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestamp_end+tol*1e9))
    else:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestampstart+tol*1e9))
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw=res.read()
        res.close()
    except urllib.error.URLError as e:
        print('!get_program: Error opening URL')
        print(e)
        return False,0
    else:
        prog_string=prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl=prog_list['programs'][0]
        return True, pl

def read_restdb_old(request_url):
    """
    Reads JSON data from W7-X REST API
    Returns: 
        valid: access ok
        t: numpy-array of time
        signal: numpy-array of requested signals
    by Thomsen
    """
    try:
        res = urllib.request.urlopen(request_url)
    except urllib.error.URLError as e:
        print(e)
        return False, 0, -1
    else:
        signal_raw=res.read()
        res.close()
        signal_string=signal_raw.decode(encoding='UTF-8')
        signal_list = json.loads(signal_string)
        signal0=np.array(signal_list['values'])
        t=np.array(signal_list['dimensions'])
        return True, t, signal0

def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
    """
    download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
        time in ns
    """    
    if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
        query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_LUT_"
        query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
        print("camera unknown, stopping here")
        raise Exception
    #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
    LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
    if LUTpar[0]:
        LUTid=LUTpar[2][0]['structure'][query]
        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
        if LUTs[0]:
            LUTs=LUTs[2][0]
            LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True,LUT
        else:
            print("unable to download the LUTs")
            del LUTpar, LUTs
            return False,0
    else:
        del LUTpar
        print("unable to find LUTs, check your request")
        return False,0

def download_NUC_by_program(port,program,exposure,version=1):
    prog=AKF_1.get_program_from_PID(program)
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_NUC_by_times(port,starttime,stoptime,exposure,version)
    else:
        print("cannot find the program")
        return False,0,0

def download_NUC_by_times(port,starttime,stoptime,exposure,version=1):
    larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_NUC_"
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        
        n=0
        nuctimes=[0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp']==exposure:
                nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i=NUCpar['structure']['offset_index']                
            n+=1 
        if nuctimes[0]!=0:
            NUC=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
            if NUC[0]:
                images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)                    
                return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
            else:
                print("NUC image for requested exposure time not found")
                return False, 0, 0
        else:
            print("NUC image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
    
def download_background_by_program(port,program,exposure,version=1):
    prog=AKF_1.get_program_from_PID(program)
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_background_by_times(port,starttime,stoptime,exposure,version)
    else:
        print("cannot find the program")
        return False,0,0
    
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=1):
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_background_"      
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_background_"   
    else:
        print("camera unknown, stopping here")
        raise Exception
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
        n=0
        backtimes=[0]
        for backpar in signal_list['values']:
            if backpar['structure']['Texp']==exposure:
                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
                    if backpar['structure']['filter']==camera_filter:
                        backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
                else:
                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
            n+=1
        if backtimes[0]!=0:
            backdat=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
            if backdat[0]:                                    
                return backdat#[True,backdat[2]]
            else:
                print("background image for requested exposure time(, filter) not found")
                return False,0,0
        else:
            print("background image for requested exposure time(, filter) not found")
            return False,0,0
    else:
        return False,0,0

def download_raw_images_by_program(port,program,time_s=0,version=1):
    prog=AKF_1.get_program_from_PID(program)
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
        if type(time_s)==list:
            tstart=time_s[0]
            tstop=time_s[1]
            if tstop<tstart:
                raise Exception("endtime before starttime")
            return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version)
        else:
            if time_s==0:
                return download_raw_images_by_times(port,starttime,stoptime,version)
            else:
                return download_raw_images_by_times(port,starttime,int(starttime+time_s*1e9),version)
    else:
        print("cannot find the program")
        return False,0,0

def download_raw_images_by_times(port,starttime,stoptime,version=1,intervalSize=1E9):
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
    else:
        print("camera unknown, stopping here")
        raise Exception
    if (stoptime-starttime)/intervalSize>1:        
        nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
        print("timewindow to large, splitting into smaller fractions ("+str(nrinterv)+")")
        intervalls=[]
        for i in range(nrinterv):
            intervalls.append(int(starttime-10+i*intervalSize))
        intervalls.append(stoptime)
        for i in range(nrinterv):
            try:
                res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1]))
                signal_list = json.loads(res.read().decode('utf-8'))
                res.close()
                
                images=[np.array(ele, dtype=np.uint16) for ele in signal_list['values']]
#                signal=np.array(signal_list['values'],dtype=np.uint16)
                t=np.array(signal_list['dimensions'])
                del signal_list
                if i==0:
                    allimages=images.copy()
                    time=t
                else:
                    allimages=allimages+images#np.append(ressignal,signal,axis=0)
                    time=np.append(time,t)
                del signal,t
            except urllib.error.URLError as e:
                print(e)                
        return True,time,allimages
    else:
        try:
            res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
            images=[np.array(ele, dtype=np.uint16) for ele in signal_list['values']]
#            signal=np.array(signal_list['values'],dtype=np.uint16)
            t=np.array(signal_list['dimensions'])
            del signal_list
            return True, t, images
        except urllib.error.URLError as e:
            print(e)
            return False, 0,-1

if fastDL:
    def download_raw_images_by_program_via_png(port,program,time_s=0,version=1,threads=1):
        prog=AKF_1.get_program_from_PID(program)
        if prog[0]:
            starttime=prog[1]['trigger']['1'][0]
            stoptime=prog[1]['trigger']['6'][0]
            success=True
            OP=get_OP_by_time(starttime)
            Cam=portcamdict[OP]['AEF'+str(port)]
            if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
                larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
            elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
                larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
            else:
                raise Exception("Port number does not fit the known cameras")
            if type(time_s)==list:
                tstart=time_s[0]
                tstop=time_s[1]
                if tstop<tstart:
                    raise Exception("endtime before starttime")
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
                stdate=stdate.isoformat()
                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
                enddate=enddate.isoformat()
            else:
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
                stdate=stdate.isoformat()
                if time_s==0:
                    enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
                    enddate=enddate.isoformat()
                else:
                    enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)  
                    enddate=enddate.isoformat()
                #"2017-11-15 08:00:00"
            times=AKF_2.get_all_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))
            time=[]
            images=[]
            lnt=len(times)
            if threads==1:
                for i in range(lnt):
                    ele=times[lnt-1-i]
                    imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
                    if imag[0]:
                        time.append(ele[0])
                        images.append(np.array(imag[1],dtype=np.uint16))
                    else:
                        success=False
                return success,np.array(time),images
            else:
                tim=[]
                for i in range(lnt):
                    tim.append(times[lnt-1-i][0])
                intervalls=[]
                intervalSize=int(lnt/threads)
                for i in range(threads):
                    intervalls.append(int(i*intervalSize))
                intervalls.append(lnt)
                jobs = []
                out_q=multiprocessing.Queue()
                for i in range(threads):
                    print("Start Thread ",i+1)                
                    p = multiprocessing.Process(target=download_raw_images_png_by_times_thread, args=(port,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,))
                    jobs.append(p)
                    p.start()
                resultdict = []
                for i in range(threads):
                    resultdict.append(out_q.get())
                for p in jobs:
                    p.join()
                print("all threads are done")
                order=[]
                for ele in resultdict:
                    order.append(ele[0])
                    if len(np.where(np.asarray(ele[1])==False)[0])>0:
                        success=False
                
                times=[]#np.array(resultdict[order.index(0)][2])
                images=[]#np.array(resultdict[order.index(0)][3])
                for i in range(threads):
                    images=images+resultdict[order.index(i)][3]
                    times=times+resultdict[order.index(i)][2]
                del resultdict
                return success,np.array(times),images
            
    
    def download_raw_images_png_by_times_thread(port,times,out_q,threadnumber,version=1):
        images=[]
        time=[]
        successes=[]
        for i in times:
            imag=download_last_raw_image_by_time(port,i-10,i+10)
            if imag[0]:
                images.append(np.array(imag[1],dtype=np.uint16))
                time.append(i)
                successes.append(True)
            else:
                successes.append(False)
        out_q.put([threadnumber,successes,time,images])        
    #    return success,np.array(time),np.array(images,dtype=np.uint16)
    
def download_last_raw_image_by_time(port,starttime,stoptime,version=1):
    OP=get_OP_by_time(starttime)
    Cam=portcamdict[OP]['AEF'+str(port)]
    if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
    elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
    else:
        raise Exception("camera unknown, stopping here")
    try:
        res = urllib.request.urlopen(larchivepath+"/_signal.png?from="+str(starttime-10)+"&upto="+str(stoptime))
        img = Image.open(BytesIO(res.read()))
        res.close()
#        pixelarray = np.array(img.getdata()).reshape(img.size[1], img.size[0])
        pixelarray = np.array(img,dtype=np.uint16)#.swapaxes(0,1)
        return True, pixelarray
    except urllib.error.URLError as e:
        print(e)
        return False, -1
    
def download_raw_parlog_by_program(port,program,version=1):
    prog=AKF_1.get_program_from_PID(program)
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
        return download_raw_parlog_by_times(port,starttime,stoptime,version)    
    else:
        print("cannot find the program")
        return False,0,0
    
def download_raw_parlog_by_times(port,starttime,stoptime,version=1):
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)
    elif port in [10,11,20,21,30,31,40,41,51]:#elif camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)   
    else:
        print("camera unknown, stopping here")
        raise Exception
    try:
        res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
        return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']
    

def get_INFRATEC_filter_by_program(program,version=1):
    prog=AKF_1.get_program_from_PID(program)
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
        return  get_INFRATEC_filter_by_times(starttime,stoptime,50,version)
    else:
        print("cannot find the program")
        return False,0,0

def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=1):
    larchivepath=archivepath+"QRT_INFRATEC/AEF"+str(port)+"_raw_PARLOG/V"+str(version)
    try:
        res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
        return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']['filter']  
    else:
        return False, 0,-1
    
def get_exposure_by_program(port,program,version=1):
    prog=AKF_1.get_program_from_PID(program)
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
        return get_exposure_by_times(port,starttime,stoptime,version)
    else:
        print("cannot find the program")
        return False,0,0
        
def get_exposure_by_times(port,starttime,stoptime,version=1):
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"   
    return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/3/exposuretime"+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
      
def get_camera_temp_by_program(port,program,version=1):
    prog=AKF_1.get_program_from_PID(program)
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
        return get_camera_temp_by_times(port,starttime,stoptime,version)
    else:
        print("cannot find the program")
        return False,0,0
        
def get_camera_temp_by_times(port,starttime,stoptime,version=1):
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
    else:
        print("Port unknown")
        raise Exception
    dummy=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if dummy[0]:
        return dummy
    else:
        return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratempetarure/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    
def get_sensor_temp_by_program(port,program,version=1):
    prog=AKF_1.get_program_from_PID(program)
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
        return get_sensor_temp_by_times(port,starttime,stoptime,version)
    else:
        print("cannot find the program")
        return False,0,0
        
def get_sensor_temp_by_times(port,starttime,stoptime,version=1):
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
    return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))

def get_camera_framerate_by_program(port,program,version=1):
    prog=AKF_1.get_program_from_PID(program)
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
        return get_camera_framerate_by_times(port,starttime,stoptime,version)
    else:
        print("cannot find the program")
        return False,0,0
        
def get_camera_framerate_by_times(port,starttime,stoptime,version=1):
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
    return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    

def get_average_divertor_TC_for_camtemp(port,starttime,endtime,part="all",TC=0):
    urlstart="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.115/"
    if port==50 or port==51:
        urlstart=urlstart+"DataModuleDesc.19471_DATASTREAM/"
    else:
        urlstart=urlstart+"DataModuleDesc.262_DATASTREAM/"
    if port %10==1:##uneven cameraport, even TC port
        portnr=port-1
        offset=(port//10-1)*32
    elif port%10==0:##even cameraport, uneven TC port
        portnr=port+1
        offset=(port//10-1)*32+16
    else:
        print("unknown port!")
        raise Exception
    if portnr==50:
        offset=0
    elif portnr==51:
        offset=16
    ext="/ActV1QRD"+str(portnr)+"CT1"
    n=0
    signal=0
    success=True
    if portnr==21:
        skip=True
        skipnr=1
    else:
        skip=False
        skipnr=0
    if part=="all":
        rs=0
        re=10
    elif part in ["h","horizontal","full h","full horizontal"]:
        rs=0
        re=7
    elif part in ["v","vertical","vertikal"]: 
        rs=7
        re=10
    elif part in ["hli","horizontal low iota"]:
        rs=0#4
        re=4#7
    elif part in ["hhi","horizontal high iota","high iota","hi"]:
        rs=4#0
        re=7#4
    elif part in [0,1,2,3,4,5,6,7,8,9,10]:
        rs=part
        re=part+1
    elif part in ["Single","s","single"]:
        rs=TC
        re=TC+1
    else:
        rs=0
        re=0
    if rs!=re:
        for i in range(rs,re,1):
            if skip and i==skipnr:
                i+=1
            url=urlstart+str(offset+i)+ext+"0"+str(i)+"_TC"+str(i%8+1)
            try:
                result=AKF_1.read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
    #                res.append(result)
                if result[0]:
                    n+=1            
                    time=result[1]
                    signal=signal+np.array(result[2])
                else:
                    time=0
                    success=False
            except Exception as E:
                print(E)        
                success=False
        try:
            return success,time,signal/n
        except:
            return False,0,-1
    else:
        return False,0,-1

def get_TC_data_for_port_by_program(port,program,part=0,version=0):
    prog=AKF_1.get_program_from_PID(program)
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return get_TC_data_for_port(port,starttime,stoptime,part,version)
    else:
        print("cannot find the program")
        return False,0,0
        
def get_TC_data_for_port(port,starttime,endtime,part=0,version=0):
    urlstart="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.115/DataModuleDesc.19471_DATASTREAM/"
    #66,67(10),72,73(20),78,79(30),84,85(40/50_2),90,91(50_1)
    TC_port={
    10:[10],
    11:[10],
    20:[20],
    21:[20],
    30:[30],
    31:[30],
    40:[10,20,30],
    41:[10,20,30],
    50:[50],
    51:[10,20,30]
    }
    TC_channel={
    10:[[66,67],["ActV1QSR10CT001","ActV1QSR10CT002"]], 
    20:[[72,73],["ActV1QSR20CT001","ActV1QSR20CT002"]],
    30:[[78,79],["ActV1QSR30CT001","ActV1QSR30CT002"]],
    50:[[90,91,84,85],["ActV1QRT50CT001","ActV1QRT50CT002","ActV1QRT50CT003","ActV1QRT50CT004"]]
    }
    n=0
    success=True
    signal=0
    time=0
    try:
        for thisport in TC_port[port]:
            thisTCchannel=TC_channel[thisport]
            if part==0:
                for chi in range(len(thisTCchannel[0])):            
                    url=urlstart+str(thisTCchannel[0][chi])+"/"+thisTCchannel[1][chi]
                    try:
                        result=AKF_1.read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
                        if result[0]:
                            n+=1            
                            time=result[1]
                            signal=signal+np.array(result[2])
                        else:                     
                            success=False
                    except Exception as E:
                        print(E)
                        success=False
                        n=1
            else:
                try:
                    url=urlstart+str(thisTCchannel[0][part-1])+"/"+thisTCchannel[1][part-1]
                    try:
                        result=AKF_1.read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
                        if result[0]:
                            n+=1            
                            time=result[1]
                            signal=signal+np.array(result[2])
                        else:                     
                            success=False
                            n=1
                    except Exception as E:
                        print(E)
                        success=False
                        n=1
                except Exception as E:
                    print(E)
                    success=False
                    n=1                    
    except:
        success=False
        n=1
    return success,time,signal/n
"""
end of download functions, below combination to get temperature
"""

#def apply_LUT_to_images(LUT,images):
#    LUTd={LUT[0][i]: LUT[1][i] for i in range(0, len(LUT[0]))}
#    return np.vectorize(LUTd.get)(images)

def temp_from_LUT(LUT,images):    
    return LUT[0,images]

def terror_from_LUT(LUT,images):
    return LUT[1,images]

def check_temperature_range(time,maxvalue_C,minvalue_C,port,exposure,cfilter=0):
    try:
        cam=portcamdict[get_OP_by_time(time)]['AEF'+str(port)]
        ranges=camlimdict[cam][exposure]
#        if minvalue_C>=ranges[0] and maxvalue_C<=ranges[1]:
#            return True
        if minvalue_C<ranges[0] and maxvalue_C<=ranges[1]:
            raise Exception("min value ({0:.2f}°C) is out of the valid temperature range (T>{1:.2f}°C)".format(minvalue_C,ranges[0]))
#            return False
        elif minvalue_C>=ranges[0] and maxvalue_C>ranges[1]:
            raise Exception("max value ({0:.2f}°C) is out of the valid temperature range (T<{1:.2f}°C)".format(maxvalue_C,ranges[1]))
#            return False
        elif minvalue_C<ranges[0] and maxvalue_C>ranges[1]:
            raise Exception("max and min value ({0:.2f}°C,{1:.2f}°C) is out of the valid temperature range ({2:.2f}°C<T<{3:.2f}!C)".format(maxvalue_C,minvalue_C,ranges[0],ranges[1]))
#            return False
    except Exception as E:
        raise Exception(E)
        

def get_temp_from_raw_by_program_V1(portnr,program,time_s=0,emi=0.8,divertorpart="all",version=1,intervalSize=1E9,give_ERROR=False):
    background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,1,version)    
    prog=AKF_1.get_program_from_PID(program)
    if prog[0]:
#        t0=prog[1]['trigger']['0'][0]
        t1=prog[1]['trigger']['1'][0]
        t6=prog[1]['trigger']['6'][0]
        starttime=t1-10
        if type(time_s)==list:
            tstart=time_s[0]
            tstop=time_s[1]
            if tstop<tstart:
                raise Exception("endtime before starttime")
            stoptime=int(starttime+tstop*1e9)  
            starttime=int(starttime+tstart*1e9)        
        else:
            if time_s==0:                                
                stoptime=t6
            else:
                stoptime=int(starttime+time_s*1e9)
        success=True
        print(datetime.datetime.now(),"Start download of raw images")
        if (stoptime-starttime)/intervalSize>1:        
            nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
            print("timewindow to large, splitting into smaller fractions ("+str(nrinterv)+")")
            intervalls=[]
            for i in range(nrinterv):
                intervalls.append(int(starttime-10+i*intervalSize))
            intervalls.append(stoptime)
            temperatureimages=[]
            times=[]
            for i in range(nrinterv):                            
                raw_dl=download_raw_images_by_times(portnr,intervalls[i],intervalls[i+1],version,intervalSize)
                print(datetime.datetime.now(),"download of raw images part "+str(i+1)+" of "+str(nrinterv)+" finished")
                if raw_dl[0]:
                    time=raw_dl[1]
                    images=raw_dl[2]
                    del raw_dl
#                    images=images.swapaxes(1,2)
                    images=[im.swapaxes(0,1) for im in images]
                    images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,True)                                        
                    temperatureimages=temperatureimages+images
                    times=times+time                                        
                    del time,images
                else:
                    raise Exception("cannot download the raw images")
        else:
            raw_dl=download_raw_images_by_times(portnr,starttime,stoptime,version,intervalSize)
            if raw_dl[0]:
                times=raw_dl[1]
                temperatureimages=raw_dl[2]
                del raw_dl
#                temperatureimages=temperatureimages.swapaxes(1,2)
                temperatureimages=[im.swapaxes(0,1) for im in temperatureimages]
                if give_ERROR:
                    temperatureimages,error_images=apply_calib_on_raw(temperatureimages,background,LUT,refT,gain,offset,gain_error,offset_error,True,give_ERROR)                                    
                else:
                    temperatureimages=apply_calib_on_raw(temperatureimages,background,LUT,refT,gain,offset,gain_error,offset_error,True,give_ERROR)                                    
            else:
                raise Exception("cannot download the raw images")
        valid=True
        for i in range(len(temperatureimages)):
            try:
                check_temperature_range(times[0],np.max(temperatureimages)-273.15,np.min(temperatureimages)-273.15,portnr,t_exp,cfilter)
            except Exception as E:
                print(E)
                valid=False
                i=len(temperatureimages)
        if give_ERROR:
            return success,times,temperatureimages,valid,error_images
        else:
            return success,times,temperatureimages,valid          
    else:
        print("cannot find program")
        return False,0,-1,False

def get_temp_from_raw_by_program_V2(portnr,program,time_s=0,emi=0.8,version=1,threads=1,give_ERROR=False):
    return get_temp_from_raw_by_program(portnr,program,time_s=time_s,emi=emi,T_version=2,version=version,threads=threads,give_ERROR=give_ERROR)
  
def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,version=1,threads=1,give_ERROR=False):
    background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version)
    if fastDL: 
        exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_s,version,threads)
    else:
        exist,time,frames=download_raw_images_by_program(portnr,program,time_s,version)
        if exist:
            frames=[im.swapaxes(0,1) for im in frames]
        else: