downloadversionIRdata.py

# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018
last update on Fr Nov 23 15:37:00 2018

Version: 3.5.0
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3)
            .
            #of updates to add functionalities
            .
            #number of updates for bug fixes )
@author: holn
"""
version = "V3.5.0"

import numpy as np
import IR_image_tools as IR_tools
from IR_da_config_constants import archivepath, testarchivepath, portcamdict, camlimdict, \
                                valid_FOV_circle, valid_background_rectangle, \
                                TC_port, TC_channel, IRCamColdframes_fittingpath, \
                                exJet, portpathdict, project, project_ana, \
                                heatflux_requestlist_path#, exJet_trans
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
import datetime
import urllib.request
import urllib.error
import logging
import json
from PIL import Image
from io import BytesIO
from os.path import join
import matplotlib.pyplot as plt
try:
    import threading
    import w7xarchive as AKF_2
    # disbale deprecation warnings, because of timezone warning spam
    import warnings
    warnings.filterwarnings("ignore", category=DeprecationWarning)
    fastDL = True
except Exception as E:
    print(E)
    fastDL = False


try:
    config_path = "\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\"\
    "Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
    FOV = plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:, :, 0]
    del FOV
except:
    config_path = ""


def get_started():
    """
    call it to get a short introduction to the available functions in this library\n
    it opens a weblink to a presentation and as well a flow chart
    """
    print("Welcome")
    link = "https://wikis.ipp-hgw.mpg.de/PhysicsW7X/images/9/9f/2019-04-24_20190424_How_to_access_Infrared_Data.pdf"
    path=str(repr(__file__)).split("downloadversionIRdata")[0].split("'")[1]
    image = path + "IR_data_access_flow_chart.png"
    try:
        import webbrowser
        webbrowser.open(link, new=2)
        try:
            os.startfile(image)
        except AttributeError:
            # Non-windows os:
            webbrowser.open(image)
    except Exception:
        # When ever that fails, just print the links:
        print("Some slides can be found in the wiki:")
        print(link)
        plt.figure()
        im=plt.imread(image)
        plt.imshow(im)        
        print("And a flow-chart for getting and using the data can be found here:")
        print(image)
        plt.show()

    

def get_latest_version(stream, project="W7X", testmode=False, t_from=None,
                       t_to=None, program=None, verbose=0):
    """Find out latest version of given stream

     INPUT
     ------
         stream: string
             url stream of interest
         project: string, optional, default 'W7X'
             the main tree in the database
         testmode: boolean, optional, default False
             whether ArchiveDB oder Test Archive should be used for the request
         t_from: uint64, optional
             timestamp in nanosecond from where the data version should be identified
         t_to: uint64, optional
             timestamp in nanosecond up to where the data version should be identified
         program: string, optional
             program id as a string in the form of "yyyymmdd.pid"
         verbose: integer, optional, default=0
             defines the output level, 1 only this function, 2 this and one level below functions etc.
     RESULT
     ------
         version: integer or None
             return int of version number of None if non-versioned or non-existing stream
     author: G. Schlisio, edit by holn
     """
    if testmode:#check which archive should be used
        base = testarchivepath
    else:
        base = archivepath

    if program == None and t_from == None and t_to == None:# check wheather any time information was given
        request = urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})
    else:
        if program is not None:
#             prog=get_program_from_PID(program)
#             if prog[0]:
#                 t_from=prog[1]['trigger']['0'][0]
#                 t_to=prog[1]['trigger']['6'][0]
            exist, t_from, _, t_to = get_trigger_from_PID(program, None, testmode, verbose=verbose-1)
            if not exist:
                raise Exception("Cannot identify the Program")

        elif t_from is not None and t_to == None:
            t_to = int(t_from+100e9)
        elif t_from == None and t_to is not None:
            t_from = int(t_to-10e9)
        request = urllib.request.Request(base +project+"/" + stream + "/_versions.json?from="+str(t_from)+"&upto="+str(t_to), headers={"Accept": "application/json"})
    try:
        response = urllib.request.urlopen(request)
        d = json.loads(response.read().decode('utf-8'))
    except urllib.error.HTTPError as ex:
        msg = ex.read()
        raise RuntimeError(msg)
    except Exception as E:
        raise RuntimeError(E)
    else:
        response.close()
     # detect unversioned or non-existing stream
    if d["versionInfo"] == []:
        return None
    versions = []
    for i in d['versionInfo']:
        versions.append(i['number'])

    return max(versions)

def TimeToNs(date, time):
    """
    TimeToNs(date, time) 
    
    INPUT
    ------
         data: list
             [year, month, day]
         time: list
             [hours, minutes, seconds, microseconds]
    RESULT
    ------
         nsdate: integer
             nanosecond since 1.1.1970 0:00:00
    """
    date_time = datetime.datetime(date[0], date[1], date[2], time[0], time[1],
                                  time[2], time[3])
    div = date_time-datetime.datetime(1970, 1, 1, 0, 0, 0)
    nsdate = div.total_seconds()*1e9
    return int(nsdate)

def read_program(timestamp_start, timestamp_end=0, tol=60):
    """
    read_program() downloads the information from the W7-X ArchiveDB and provides the programnumber and triggers
    
    INPUT
    ------
        timestamp_start: integer
            start time in nanoseconds
        timestamp_end: inter, optional, default is 0
            time to stop the search in nanoseconds
        tol: integer, optional, default is 60
            toleranz in seconds around the timestamp_start for searching the program in the W7-X database
    RESULT
    ------
        exist: boolean
            True if data was found, False if not
        program: dictionary
            W7-X program information
    
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
    if timestamp_end == 0:
        timestamp_end = timestamp_start
    jsonurl = '{0}={1}&upto={2}'.format(program_url,
                                        int(timestamp_start-tol*1e9),
                                        int(timestamp_end+tol*1e9))
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw = res.read()
        res.close()
    except urllib.error.URLError as e:
        print('read_program: Error opening URL')
        print(e)
        return False, 0
    except Exception as e:
        print('read_program: Error opening URL', e)
        return False, 0
    else:
        prog_string = prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl = prog_list['programs'][0]
        return True, pl



def download_LUT(port, time, exposure=0, emissivity=0, camera_filter=0, version=0,
                 testmode=False, verbose=0):
    """
    download_LUT(port, time, exposure=0, emissivity=0, camera_filter=0, version=1):
        download of the look up table for the infrared cameras from the database for OP1.2(a+b)
    Have to swap 11, 21 until correction in the database
    INPUT
    ------
        port: interger
            number of the AEF camera port
        time: integer
            nanosecond timestamp for the LUT            
        exposure: integer
            requested exposure time in microseconds
        emissivity: float, string or integer
            emissivity value to identify which LUT is requested
        camera_filter: inter or string, only needed for port 50
            filter number of the Infratec camera in port AEF50
        version: integer, optional, default 0
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional, default False
            if True, the data will be loaded from the test archive
        verbose: integer, optional, default 0
            feedback level (details of print messages)     
    RESULT
    ------
        exist: boolean
            True if Look up table was found, False if not
        LUT: list
            contains the calibration information: [digital level, temperature, temperature error]
    NOTE
    ------
        function need to be adapted for OP2, if AEK ports come into opperation
    """
    OP = IR_tools.get_OP_by_time(time_ns=time)#getting operation phase
    if port == 21:# swapping both ports (21 and 11)
        port = 11
    elif port == 11:
        port = 21
    if testmode: #check which archive should be used
        larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
    else:
        larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
    #camera dependend request string generation
    if port == 50 and OP == "OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        query = "Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    elif port in [10, 11, 20, 21, 30, 31, 40, 41, 51] or OP == "OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        query = "Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
#        print("download_LUT: Error! Camera unknown, stopping here.")
        logging.warning("download_LUT: Error! Camera unknown, stopping here.")
        return False, 0
#        raise Exception
    if version == 0:#version check, if 0
        version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_PARLOG", t_from=time, testmode=testmode)
    if verbose > 0:
        print("download_LUT: LUT V"+str(version)+" is used")
    #time=int(fu.TimeToNs([2017,9,26], [8, 0, 0, 0]))
    LUTpar = read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
    if LUTpar[0]:
        LUTid = LUTpar[2][0]['structure'][query]
        #getting here now the time from the parlog
        Timetamp = LUTpar[2][0]['structure']['timestamp']
        LUTs = read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(Timetamp-10)+"&upto="+str(Timetamp+20))
        if LUTs[0]:
            LUTs = LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
            LUT = [LUTs[0], LUTs[LUTid], LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True, LUT
        else:
            print("download_LUT: Warning! unable to download the LUTs")
            del LUTpar, LUTs
            return False, 0
    else:
        del LUTpar
        print("download_LUT: Warning! unable to find LUTs, check your request")
        return False, 0



def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, verbose=0):#emissivity_steel=0.31,
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)

    Read LUT V3 from local files for testing new calibration.
    INPUT
    ------
        port: integer or string
            the port number of the AEF port        
        this_time: integer
            nanosecond timestamp for the LUT            
        t_exp: integer
            requested exposure time in microseconds
        emissivity: float, string or integer
            emissivity value to identify which LUT is requested
        cfilter: inter or string, only needed for port 50
            filter number of the Infratec camera in port AEF50
        verbose: integer, optional, default 0 
            feedback level (details of print messages) 
    RESULT
    ------
        exist: boolean
            True if Look up table was found, False if not
        LUT: list
            contains the calibration information: [digital level, temperature, temperature error]
    NOTE
    ------
        function need to be adapted for OP2, if AEK ports come into opperation
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
    OP = IR_tools.get_OP_by_time(time_ns=this_time)
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
        with open(LUT_dir+"\\"+filename) as data_file:
            jfile = json.load(data_file)
        LUT = np.array(jfile['LUT']).swapaxes(0, 1)
        if verbose > 0:
            print("read_LUT_from_file: succesfully loaded V3 LUT from local directory")
        return True, LUT
    except Exception as E:
        print("read_LUT_from_file: ERROR in loading V3 LUTs", E)
        return False, []

def download_NUC_by_program(port, program, exposure, version=0, testmode=False, verbose=0):
    '''
    download the NUC data for a given port, program and exposure time of the camera
    INPUT
    ------
        port: integer or string
            port number of the AEF port of the camera
        program: string
            program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016'
        exposure: integer
            exposure time in microseconds
        version: integer, optional, default 0
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional, default False
            if True, the data will be loaded from the test archive
        verbose: integer, optional, default 0
            feedback level (details of print messages) 
    RESULT
    ------
        exist: boolean
            True if NUC data was found, False if not
        NUC: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        NUC_describtion: list
            a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
    NOTE
    ------
        function need to be adapted for OP2, if AEK ports come into opperation
    '''
#    prog=get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
#        starttime=prog[1]['trigger']['0'][0]
#        stoptime=prog[1]['trigger']['1'][0]
    exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
    if exist:
        return download_NUC_by_times(port, starttime, stoptime, exposure, version, verbose=verbose)
    else:
#    except:
        print("download_NUC_by_program: cannot find the program")
        return False, 0, 0

def download_NUC_by_times(port, starttime, stoptime, exposure, testmode=False, version=0, verbose=0):
    '''
    download the NUC data for a given port, start and endtime in ns and exposure time of the camera
    INPUT
    ------
        port: integer or string
            port number of the AEF port of the camera
        starttime: integer
            time in ns where the search should start
        stoptime: integer
            time in sn where the search should end
        exposure: integer
            exposure time in microseconds
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive
        verbose: integer, optional    
            feedback level (details of print messages) 
    RESULT
    ------
        exist: boolean
            True if NUC data was found, False if not
        NUC: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        NUC_describtion: list
            a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
    NOTE
    ------
        function need to be adapted for OP2, if AEK ports come into opperation
    '''
    OP = IR_tools.get_OP_by_time(time_ns=starttime)
    if testmode:
        larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
    else:
        larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version == 0:
        version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM", t_from=starttime, testmode=testmode)
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon = True
    except urllib.error.URLError as e:
        print('download_NUC_by_times: Error! ', e)
        goon = False
    except Exception as e:
        print('download_NUC_by_times: Error! ', e)
        goon = False
    if goon:
        n = 0
        nuctimes = [0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp'] == exposure:
                nuctimes = [signal_list['dimensions'][2*n], signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i = NUCpar['structure']['offset_index']
            n += 1
        if nuctimes[0] != 0:
            NUC = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
            if NUC[0]:
                images = np.vsplit(NUC[2], np.shape(NUC[2])[0]/offset_i)
                return True, images, ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
            else:
                if verbose > 0:
                    print("download_NUC_by_times: NUC image for requested exposure time not found")
                return False, 0, 0
        else:
            if verbose > 0:
                print("download_NUC_by_times: NUC image for requested exposure time not found")
            return False, 0, 0
    else:
        return False, 0, 0

def get_NUC_by_program(port, program, exposure, version=0, testmode=False, verbose=0):
    '''
    Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera
    in a given port and time interval. Depending on time and camera, reconstruct
    the cold frame and/or compute the gain, offset from the hot and cold frame.
    INPUT
    ------
        port: integer or string
            port number of the AEF port of the camera
        program: string
            program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016'
        exposure: integer
            exposure time in microseconds
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive
        verbose: integer, optional    
            feedback level (details of print messages) 
    RESULT
    ------
        exist: boolean
            True if NUC data was found, False if not
        NUC: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        NUC_describtion: list
            a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
    NOTE
    ------
        function need to be adapted for OP2, if AEK ports come into opperation
    '''
#    prog=get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
#        starttime=prog[1]['trigger']['0'][0]
#        stoptime=prog[1]['trigger']['1'][0]
    exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
    if exist:
        return get_NUC_by_times(port, starttime, stoptime, exposure, version=version, testmode=testmode, verbose=verbose)
    else:
#    except:
        print("get_NUC_by_program: Warning! cannot find the trigger timestamps for program", program)
#        return False, 0, 0
        OP = IR_tools.get_OP_by_time(program_str=program)
        return get_NUC_by_times(port, starttime, stoptime, exposure, version=version, testmode=testmode, OP=OP, verbose=verbose)

def get_NUC_by_times(port, starttime, stoptime, t_exp, version=0, testmode=False, OP=None, verbose=0):
    """
    Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera
    in a given port and time interval. Depending on time and camera, reconstruct
    the cold frame and/or compute the gain, offset from the hot and cold frame.
    the result is operation phase dependent, due to changes in the aquisition and saving programs
    
    INPUT
    ------
        port: integer or string
            port number of the AEF port of the camera
        starttime: integer
            time in ns where the search should start
        stoptime: integer
            time in sn where the search should end
        exposure: integer
            exposure time in microseconds
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive
        OP: string, optional, default None
            name of operation phase
        verbose: integer, optional    
            feedback level (details of print messages) 
    RESULT
    ------
        exist: boolean
            True if NUC data was found, False if not
        NUC: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        NUC_describtion: list
            a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
    NOTE
    ------
        function need to be adapted for OP2, if AEK ports come into opperation
    """
    if OP is None: #get operation phase if not given
        OP = IR_tools.get_OP_by_time(time_ns=stoptime)
    if OP == "OP1.2a": # check for operation phase
        t1 = stoptime
        t0 = starttime
        prog = read_program(t1)
        program = prog[1]['id']
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):#camera 31 and 21 need after 20171110 new coldframes due to a open shutter during the recording of the coldframes
            if verbose > 0:
                print("get_NUC_by_times: rebuilding coldframe")
            #use any way the rebuilt coldframe.
            sT = get_sensor_temp_by_program(port, program, testmode=testmode)[2][0]
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit, hotcold, describtion = download_hot_cold_reference_by_times(port, t_exp, testmode=False)
            del describtion
            if gotit:
                coldref = hotcold[1]
                hotref = hotcold[0]
            else:
#                raise Exception("get_NUC_by_times: unable to download reference frames")
                logging.warning("get_NUC_by_times: unable to download reference frames")
                return False, 0, 0
            filestring = 'AEF' + str(port) + '_et' + str(int(t_exp))
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
            if verbose > 0:
                print('get_NUC_by_times: calculate gain, offset and bad pixels')
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref, verbose=verbose-1)
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1)
            gain_error = 0
            offset_error = 0
            return True, [gain, offset, cirebuild, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
        elif (port == 11) or (port == 21): # case of the swaped cameras
            if verbose > 0:
                print('get_NUC_by_times: downloading NUC')
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit, hotcold, describtion = download_hot_cold_reference_by_times(port, t_exp, testmode=False)
            if gotit:
                coldref = hotcold[1]
                hotref = hotcold[0]
            else:
#                raise Exception("get_NUC_by_times: unable to download reference frames")
                logging.warning("get_NUC_by_times: unable to download reference frames")
                return False, 0, 0
            NUC_DL = download_NUC_by_times(port, t0, t1, t_exp, version, testmode)
            if NUC_DL[0] == False:
#                raise Exception("get_NUC_by_times: NUC was not found")
                logging.warning("get_NUC_by_times: NUC was not found")
                return False, 0, 0
            else:
                ci = NUC_DL[1][2]
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref, verbose=verbose-1)
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1)
            gain_error = 0
            offset_error = 0
            return True, [gain, offset, ci, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
        else: #all other cases, the "normal" case
            if verbose > 0:
                print('get_NUC_by_times: downloading NUC')
            NUC_DL = download_NUC_by_times(port, starttime, stoptime, t_exp, testmode=testmode)
            #True, images, ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
            if NUC_DL[0]:
                badpixels = find_badpixels(port,NUC_DL[1][0],NUC_DL[1][1],niterations=10, tolerance=10, verbose=verbose-1)
                NUC_DL[1][3]=badpixels
            return NUC_DL
    elif OP == "OP1.2b":
        gain_error = 0
        offset_error = 0
        gotit, hotcold, describtion = download_hot_cold_reference_by_times(port, t_exp, testmode=False)
        if gotit:
            coldref = hotcold[1]
            hotref = hotcold[0]
        else:
#            raise Exception("get_NUC_by_times: unable to download reference frames")
            logging.warning("get_NUC_by_times: unable to download reference frames")
            return False, 0, 0
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
        try:
            exist, _, coldframes = download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, frametype=0, version=0, testmode=testmode, verbose=verbose-1)
        except:
            exist = False
        if exist:#okay it exist, average the cold frames
            cold = np.zeros(np.shape(coldframes[0]), dtype=np.uint64)
            for ele in coldframes:
                cold += ele
            cold = np.asarray(cold/len(coldframes), dtype=np.uint16)
        else:
            print('get_NUC_by_times: Warning! no cold frames found. will use reference cold frame...')
            cold = np.copy(coldref)
#            return False, [0], [0]
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref, verbose=verbose-1)
        badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1)
        return True, [gain, offset, cold, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
    else:
#        raise Exception("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP")
        logging.warning("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP")
        return False, 0, 0

def download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, frametype=0, version=0, testmode=False, verbose=0):
    """
    download of the calibration raw files which are recorded between t0 and t1 trigger
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
    
    INPUT
    ------
        port: integer 
            port number of the AEF port of the camera
        t_exp: integer
            exposure time in microseconds
        starttime: integer
            time in ns where the search should start        
        stoptime: integer
            time in sn where the search should end 
        frametype: integer, default 0
            defines which type of frames should be returned, 0 for cold frames, 1 for background frames
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive        
        verbose: integer, optional    
            feedback level (details of print messages) 
    RESULT
    ------
        exist: boolean
            True if data was found, False if not
        time: list
            a list containing the time stamp of each frame
        frames: list
            a list of frames, 2D numpy arrays
    NOTE
    ------
        function need to be adapted for OP2, if AEK ports come into opperation
    """
    gotit, time_t, texp_t = get_exposure_by_times(port, int(starttime-100), int(stoptime-20e6), testmode=testmode)
    OP = IR_tools.get_OP_by_time(time_ns=stoptime)
    if gotit:
        expinds = np.where(texp_t == t_exp)[0]
        if len(expinds) == 0:
            logging.warning("download_calibration_raw_files_by_time: Error! cannot find the exposure time in the given data")
            return False, [0], [0]
    else:
        logging.warning("download_calibration_raw_files_by_time: Error! exposure time not found")
        return False, [0], [0]
    gotitf, timef, values_f = get_frametype_by_times(port, int(starttime-100), int(stoptime-20e6), testmode=testmode)
    if gotitf:
        typiinds = np.where(values_f[expinds] == frametype)[0]
        if len(typiinds) > 0:
            ref_t = [np.min(timef[expinds][typiinds]), np.max(timef[expinds][typiinds])]
#            print((ref_t[1]-ref_t[0])/1e9)
#            print(len(timef), len(timef[expinds]), len(timef[expinds][typiinds]))
        else:#okay the early data stuff or strange stuff
            if verbose > 0:
                print("download_calibration_raw_files_by_time: frame type was not identified assuming that the first part is the cold and the second one the background")
            frametimes = time_t[expinds]
            diftime = [frametimes[i]-frametimes[i-1] for i in range(1, len(frametimes))]
            turnpoint = np.where(np.asarray(diftime) > 11e6)[0][0]
            if frametype == 0:
                ref_t = [np.min(frametimes[0:turnpoint+1]), np.max(frametimes[0:turnpoint+1])]
#                print((ref_t[1]-ref_t[0])/1e9)
            elif frametype == 1:
#                print(len(frametimes[turnpoint+1:]), len(frametimes[0:turnpoint+1]), len(frametimes))
                ref_t = [np.min(frametimes[turnpoint+1:]), np.max(frametimes[turnpoint+1:])]
#                print((ref_t[1]-ref_t[0])/1e9)
            else:
#                raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
                logging.warning("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
                return False, [0], [0]
    else:
        print("download_calibration_raw_files_by_time: Error! frametype not found")
        frametimes = time_t[expinds]
        diftime = [frametimes[i]-frametimes[i-1] for i in range(1, len(frametimes))]
        turnpoint = np.where(np.asarray(diftime) > 11e6)[0][0]
        if frametype == 0:
            ref_t = [np.min(frametimes[0:turnpoint+1]), np.max(frametimes[0:turnpoint+1])]
            print((ref_t[1]-ref_t[0])/1e9)
        elif frametype == 1:
#                print(len(frametimes[turnpoint+1:]), len(frametimes[0:turnpoint+1]), len(frametimes))
            ref_t = [np.min(frametimes[turnpoint+1:]), np.max(frametimes[turnpoint+1:])]
            print((ref_t[1]-ref_t[0])/1e9)
        else:
#            raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
            logging.warning("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
            return False, [0], [0]
#        return False, [0], [0]
    t1date = datetime.datetime.utcfromtimestamp((stoptime-100)/1e9)
    t1date = t1date.isoformat()
    t0date = datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
    t0date = t0date.isoformat()
    if version == 0:
        version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM", t_from=starttime, t_to=stoptime, testmode=testmode)
    if testmode:
        larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
    else:
        larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
    timest0 = AKF_2.get_time_intervals(larchivepath, t0date.replace("T", " "), t1date.replace("T", " "))
    t_offset = timest0[-1][0]-time_t[0]
    if t_offset > 10e6:
        if verbose > 0:
            print("download_calibration_raw_files_by_time: time offset detected, try to correct this, offset is", t_offset)
    else:
        t_offset = 0
#            print("starttime frames:", np.min(timest0), "starttime metachannels:", time_t[0], "offset", t_offset)
    return download_raw_images_by_times(port, ref_t[0]+t_offset, ref_t[1]+t_offset, testmode=testmode, verbose=verbose-1)

def download_hot_cold_reference_by_times(port, exposure, starttime=1503907200000000000,
                                         testmode=False, version=0):
    """
    Loads the most recent hot and cold calibration frames for a starttime.
    Uses first calibration frames if time is not defined.
    INPUT
    ------
        port: integer or string
            port number of the AEF port of the camera, or string containing the full port name
        exposure: integer
            exposure time in microseconds
        starttime: integer
            time in ns where the search should start
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive        
        verbose: integer, optional    
            feedback level (details of print messages) 
    RESULT
    ------
        exist: boolean
            True if data was found, False if not
        images: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        image_describtion: list
            a list which entry in NUC is what, looks like ['hot','cold']
    NOTE
    ------
        function need to be adapted for OP2, if AEK ports come into opperation
    """
    OP = IR_tools.get_OP_by_time(time_ns=starttime)
    if isinstance(port, int):
        port = "AEF"+str(port)
    if testmode:
        larchivepath = testarchivepath+project+"/"+portpathdict[OP][port]+"raw_"
    else:
        larchivepath = archivepath+project+"/"+portpathdict[OP][port]+"raw_"
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version == 0:
        version = get_latest_version(portpathdict[OP][port]+"raw_DATASTREAM", t_from=starttime)
    try:
        path_string = larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9))
        res = urllib.request.urlopen(path_string)
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon = True
    except urllib.error.URLError as e:
        print('download_hot_cold_reference_by_times: Error! ', e)
        goon = False
    except Exception as e:
        print('download_hot_cold_reference_by_times: Error! ', e)
        goon = False
    if goon:
        COLDtime = 0
        HOTtime = 0
        try:
            COLDtime = signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
        except:
            print("download_hot_cold_reference_by_times: cold image for requested exposure time not found")
        try:
            HOTtime = signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
        except:
            print("download_hot_cold_reference_by_times: hot image for requested exposure time not found")
        images = []
        if HOTtime != 0:
            HOT = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
            if HOT[0]:
                images.append(HOT[2])
        if COLDtime != 0:
            COLD = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
            if COLD[0]:
                images.append(COLD[2])

        if HOT[0] and COLD[0] and len(images) == 2:
            return True, images, ['hot', 'cold']
        else:
            print("download_hot_cold_reference_by_times: Error! hot and cold image for requested exposure time not found")
            return False, 0, 0
    else:
        return False, 0, 0

def download_background_by_program(port, program, exposure, camera_filter=0, version=0, testmode=False, verbose=0):
    '''
    download the background from the AEF cameras in OP1.2a, which are recorded between t0 and t1 trigger
    returned exist, time, frame
    INPUT
    ------
        port: integer 
            port number of the AEF port of the camera        
        program: string
            program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016'
        exposure: integer
            exposure time in microseconds        
        camera_filter: integer or string, only needed for port 50
            filter number of the Infratec camera in port AEF50
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive        
        verbose: integer, optional    
            feedback level (details of print messages) 
		OP: string, optional, default None
            name of operation phase
    RESULT
    ------
        exist: boolean
            True if data was found, False if not
        time: list
            a list containing the time stamp of each frame
        frames: list
            a list of frames, 2D numpy arrays
    NOTE
    ------
        function need to be adapted for OP2, if AEK ports come into opperation
    '''
#    prog=get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
#        starttime=prog[1]['trigger']['0'][0]
#        stoptime=prog[1]['trigger']['1'][0]
    exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
    if exist:
        return download_background_by_times(port, starttime, stoptime, exposure, camera_filter=camera_filter, version=version)
    else:#except:
        print("download_background_by_program: Error! cannot find the program {0}".format(program))
        return False, 0, 0

def download_background_by_times(port, starttime, stoptime, exposure, camera_filter=0,
                                 version=0, testmode=False, verbose=0):
    """
    download the background frames for a given time intervall in nanoseconds, defined by start and stop time
    
    INPUT
    ------
        port: integer 
            port number of the AEF port of the camera
        starttime: integer
            time in ns where the search should start        
        stoptime: integer
            time in sn where the search should end 
        exposure: integer
            exposure time in microseconds
        camera_filter: integer or string, only needed for port 50
            filter number of the Infratec camera in port AEF50
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive        
        verbose: integer, optional    
            feedback level (details of print messages) 

    RESULT
    ------
        exist: boolean
            True if data was found, False if not
        time: list
            a list containing the time stamp of each frame
        frames: list
            a list of frames, 2D numpy arrays
    NOTE
    ------
        function need to be adapted for OP2, if AEK ports come into opperation
    """
    OP = IR_tools.get_OP_by_time(time_ns=starttime)
    if OP == "OP1.2a":
        stream = portpathdict[OP]["AEF"+str(port)]+"background_"
        if testmode:
            larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
        else:
            larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
        if version == 0:
            version = get_latest_version(stream+"DATASTREAM", t_from=starttime, testmode=testmode)
        try:
            res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
            goon = True
        except urllib.error.URLError as e:
            print('download_background_by_times: Error! ', e)
            goon = False
        except Exception as e:
            print('download_background_by_times: Error! ', e)
            goon = False
        if goon:
            n = 0
            backtimes = [0]
            for backpar in signal_list['values']:
                if backpar['structure']['Texp'] == exposure:
                    if port == 50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
                        if backpar['structure']['filter'] == camera_filter:
                            backtimes = [signal_list['dimensions'][2*n], signal_list['dimensions'][2*n+1]]
                    else:
                        backtimes = [signal_list['dimensions'][2*n], signal_list['dimensions'][2*n+1]]
                n += 1
            if backtimes[0] != 0:
                backdat = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
                if backdat[0]:
                    return backdat#[True, backdat[2]]
                else:
                    print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
                    return