downloadversionIRdata.py 281 KB
Newer Older
Holger Niemann's avatar
Holger Niemann committed
1
2
3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
4
updated on Tue Aug 21 10:20:00 2018
5
last update on Fr Nov 23 15:37:00 2018
Holger Niemann's avatar
Holger Niemann committed
6

7
Version: 3.4.1
8
9
10
11
12
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3)
            .
            #of updates to add functionalities
            .
            #number of updates for bug fixes )
Holger Niemann's avatar
Holger Niemann committed
13
14
@author: holn
"""
15
version = "V3.4.1"
Holger Niemann's avatar
Holger Niemann committed
16
17

import numpy as np
18
import IR_image_tools as IR_tools
19
20
21
22
23
from IR_config_constants import archivepath, testarchivepath, portcamdict, camlimdict, \
                                valid_FOV_circle, valid_background_rectangle, \
                                TC_port, TC_channel, IRCamColdframes_fittingpath, \
                                exJet, portpathdict, project, project_ana, \
                                heatflux_requestlist_path#, exJet_trans
24
25
26
27
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
Holger Niemann's avatar
Holger Niemann committed
28
import datetime
29
30
import urllib.request
import urllib.error
31
import logging
Holger Niemann's avatar
Holger Niemann committed
32
import json
33
34
from PIL import Image
from io import BytesIO
35
from os.path import join
36
import matplotlib.pyplot as plt
37
try:
38
    import threading
39
    import w7xarchive as AKF_2
40
41
42
    # disbale deprecation warnings, because of timezone warning spam
    import warnings
    warnings.filterwarnings("ignore", category=DeprecationWarning)
43
    fastDL = True
44
45
except Exception as E:
    print(E)
46
    fastDL = False
Holger Niemann's avatar
Holger Niemann committed
47

48

49
try:
50
51
52
    config_path = "\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\"\
    "Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
    FOV = plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:, :, 0]
53
54
    del FOV
except:
55
    config_path = ""
56

57

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
def get_latest_version(stream, project="W7X", testmode=False, t_from=None,
                       t_to=None, program=None, verbose=0):
    """Find out latest version of given stream

     INPUT
     ------
         stream: string
             url stream of interest
         project: string, optional, default 'W7X'
             the main tree in the database
         testmode: boolean, optional, default False
             whether ArchiveDB oder Test Archive should be used for the request
         t_from: uint64, optional
             timestamp in nanosecond from where the data version should be identified
         t_to: uint64, optional
             timestamp in nanosecond up to where the data version should be identified
         program: string, optional
             program id as a string in the form of "yyyymmdd.pid"
         verbose: integer, optional, default=0
             defines the output level, 1 only this function, 2 this and one level below functions etc.
     RESULT
     ------
         version: integer or None
             return int of version number of None if non-versioned or non-existing stream
82
83
     author: G. Schlisio, edit by holn
     """
84
    if testmode:#check which archive should be used
85
86
87
88
        base = testarchivepath
    else:
        base = archivepath

89
    if program == None and t_from == None and t_to == None:# check wheather any time information was given
90
91
92
        request = urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})
    else:
        if program is not None:
93
94
95
96
#             prog=get_program_from_PID(program)
#             if prog[0]:
#                 t_from=prog[1]['trigger']['0'][0]
#                 t_to=prog[1]['trigger']['6'][0]
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
            exist, t_from, _, t_to = get_trigger_from_PID(program, None, testmode, verbose=verbose-1)
            if not exist:
                raise Exception("Cannot identify the Program")

        elif t_from is not None and t_to == None:
            t_to = int(t_from+100e9)
        elif t_from == None and t_to is not None:
            t_from = int(t_to-10e9)
        request = urllib.request.Request(base +project+"/" + stream + "/_versions.json?from="+str(t_from)+"&upto="+str(t_to), headers={"Accept": "application/json"})
    try:
        response = urllib.request.urlopen(request)
        d = json.loads(response.read().decode('utf-8'))
    except urllib.error.HTTPError as ex:
        msg = ex.read()
        raise RuntimeError(msg)
    except Exception as E:
        raise RuntimeError(E)
    else:
        response.close()
116
     # detect unversioned or non-existing stream
117
118
119
120
121
    if d["versionInfo"] == []:
        return None
    versions = []
    for i in d['versionInfo']:
        versions.append(i['number'])
122

123
    return max(versions)
124

125
def TimeToNs(date, time):
Holger Niemann's avatar
Holger Niemann committed
126
    """
127
128
129
130
131
132
133
134
135
136
137
138
    TimeToNs(date, time) 
    
    INPUT
    ------
         data: list
             [year, month, day]
         time: list
             [hours, minutes, seconds, microseconds]
    RESULT
    ------
         nsdate: integer
             nanosecond since 1.1.1970 0:00:00
139
140
141
142
143
    """
    date_time = datetime.datetime(date[0], date[1], date[2], time[0], time[1],
                                  time[2], time[3])
    div = date_time-datetime.datetime(1970, 1, 1, 0, 0, 0)
    nsdate = div.total_seconds()*1e9
144
    return int(nsdate)
Holger Niemann's avatar
Holger Niemann committed
145

146
def read_program(timestamp_start, timestamp_end=0, tol=60):
Holger Niemann's avatar
Holger Niemann committed
147
    """
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
    read_program() downloads the information from the W7-X ArchiveDB and provides the programnumber and triggers
    
    INPUT
    ------
        timestamp_start: integer
            start time in nanoseconds
        timestamp_end: inter, optional, default is 0
            time to stop the search in nanoseconds
        tol: integer, optional, default is 60
            toleranz in seconds around the timestamp_start for searching the program in the W7-X database
    RESULT
    ------
        exist: boolean
            True if data was found, False if not
        program: dictionary
            W7-X program information
    
Holger Niemann's avatar
Holger Niemann committed
165
166
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
167
    if timestamp_end == 0:
168
169
170
171
        timestamp_end = timestamp_start
    jsonurl = '{0}={1}&upto={2}'.format(program_url,
                                        int(timestamp_start-tol*1e9),
                                        int(timestamp_end+tol*1e9))
Holger Niemann's avatar
Holger Niemann committed
172
173
    try:
        res = urllib.request.urlopen(jsonurl)
174
        prog_raw = res.read()
Holger Niemann's avatar
Holger Niemann committed
175
176
        res.close()
    except urllib.error.URLError as e:
177
        print('read_program: Error opening URL')
Holger Niemann's avatar
Holger Niemann committed
178
        print(e)
179
        return False, 0
180
    except Exception as e:
181
182
        print('read_program: Error opening URL', e)
        return False, 0
Holger Niemann's avatar
Holger Niemann committed
183
    else:
184
        prog_string = prog_raw.decode(encoding='UTF-8')
Holger Niemann's avatar
Holger Niemann committed
185
        prog_list = json.loads(prog_string)
186
        pl = prog_list['programs'][0]
Holger Niemann's avatar
Holger Niemann committed
187
188
        return True, pl

189

Holger Niemann's avatar
Holger Niemann committed
190

191
192
def download_LUT(port, time, exposure=0, emissivity=0, camera_filter=0, version=0,
                 testmode=False, verbose=0):
Holger Niemann's avatar
Holger Niemann committed
193
    """
194
    download_LUT(camera, port, time, exposure=0, emissivity=0, camera_filter=0, version=1):
195
        download of the look up table for the infrared cameras from the database for OP1.2(a+b)
196
    Have to swap 11, 21 until correction in the database
197
198
    INPUT
    ------
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
        port: interger
            number of the AEF camera port
        time: integer
            nanosecond timestamp for the LUT            
        exposure: integer
            requested exposure time in microseconds
        emissivity: float, string or integer
            emissivity value to identify which LUT is requested
        camera_filter: inter or string, only needed for port 50
            filter number of the Infratec camera in port AEF50
        version: integer, optional, default 0
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional, default False
            if True, the data will be loaded from the test archive
        verbose: integer, optional, default 0
            feedback level (details of print messages)     
215
216
    RESULT
    ------
217
218
219
220
        exist: boolean
            True if Look up table was found, False if not
        LUT: list
            contains the calibration information: [digital level, temperature, temperature error]
221
222
    NOTE
    ------
223
        function need to be adapted for OP2, if AEK ports come into opperation
224
    """
225
226
    OP = IR_tools.get_OP_by_time(time_ns=time)#getting operation phase
    if port == 21:# swapping both ports (21 and 11)
227
228
229
        port = 11
    elif port == 11:
        port = 21
230
    if testmode: #check which archive should be used
231
        larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
232
    else:
233
        larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
234
    #camera dependend request string generation
235
236
237
238
    if port == 50 and OP == "OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        query = "Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    elif port in [10, 11, 20, 21, 30, 31, 40, 41, 51] or OP == "OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        query = "Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
Holger Niemann's avatar
Holger Niemann committed
239
    else:
240
241
242
243
244
#        print("download_LUT: Error! Camera unknown, stopping here.")
        logging.warning("download_LUT: Error! Camera unknown, stopping here.")
        return False, 0
#        raise Exception
    if version == 0:#version check, if 0
245
246
        version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM", t_from=time, testmode=testmode)
    if verbose > 0:
247
        print("download_LUT: LUT V"+str(version)+" is used")
248
249
    #time=int(fu.TimeToNs([2017,9,26], [8, 0, 0, 0]))
    LUTpar = read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
250
    if LUTpar[0]:
251
252
        LUTid = LUTpar[2][0]['structure'][query]
        LUTs = read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
253
        if LUTs[0]:
254
255
            LUTs = LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
            LUT = [LUTs[0], LUTs[LUTid], LUTs[LUTid+1]]
Holger Niemann's avatar
Holger Niemann committed
256
            del LUTpar, LUTs
257
            return True, LUT
Holger Niemann's avatar
Holger Niemann committed
258
        else:
259
            print("download_LUT: Warning! unable to download the LUTs")
Holger Niemann's avatar
Holger Niemann committed
260
            del LUTpar, LUTs
261
            return False, 0
Holger Niemann's avatar
Holger Niemann committed
262
263
    else:
        del LUTpar
264
        print("download_LUT: Warning! unable to find LUTs, check your request")
265
266
        return False, 0

Holger Niemann's avatar
Holger Niemann committed
267

268

269
def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, verbose=0):#emissivity_steel=0.31,
270
271
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
272

273
    Read LUT V3 from local files for testing new calibration.
274
275
    INPUT
    ------
276
277
278
279
280
281
282
283
284
285
286
287
        port: integer or string
            the port number of the AEF port        
        this_time: integer
            nanosecond timestamp for the LUT            
        t_exp: integer
            requested exposure time in microseconds
        emissivity: float, string or integer
            emissivity value to identify which LUT is requested
        cfilter: inter or string, only needed for port 50
            filter number of the Infratec camera in port AEF50
        verbose: integer, optional, default 0 
            feedback level (details of print messages) 
288
289
    RESULT
    ------
290
291
292
293
        exist: boolean
            True if Look up table was found, False if not
        LUT: list
            contains the calibration information: [digital level, temperature, temperature error]
294
295
    NOTE
    ------
296
        function need to be adapted for OP2, if AEK ports come into opperation
297
298
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
299
    OP = IR_tools.get_OP_by_time(time_ns=this_time)
300
301
302
303
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
304
305
306
307
        with open(LUT_dir+"\\"+filename) as data_file:
            jfile = json.load(data_file)
        LUT = np.array(jfile['LUT']).swapaxes(0, 1)
        if verbose > 0:
308
            print("read_LUT_from_file: succesfully loaded V3 LUT from local directory")
309
310
        return True, LUT
    except Exception as E:
311
        print("read_LUT_from_file: ERROR in loading V3 LUTs", E)
312
        return False, []
313
314

def download_NUC_by_program(port, program, exposure, version=0, testmode=False, verbose=0):
315
    '''
316
    download the NUC data for a given port, program and exposure time of the camera
317
318
    INPUT
    ------
319
320
321
322
323
324
325
326
327
328
329
330
        port: integer or string
            port number of the AEF port of the camera
        program: string
            program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016'
        exposure: integer
            exposure time in microseconds
        version: integer, optional, default 0
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional, default False
            if True, the data will be loaded from the test archive
        verbose: integer, optional, default 0
            feedback level (details of print messages) 
331
332
    RESULT
    ------
333
334
335
336
337
338
        exist: boolean
            True if NUC data was found, False if not
        NUC: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        NUC_describtion: list
            a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
339
340
    NOTE
    ------
341
        function need to be adapted for OP2, if AEK ports come into opperation
342
343
    '''
#    prog=get_program_from_PID(program)
344
345
346
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
347
348
349
350
351
#    if prog[0]:
#        starttime=prog[1]['trigger']['0'][0]
#        stoptime=prog[1]['trigger']['1'][0]
    exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
    if exist:
352
        return download_NUC_by_times(port, starttime, stoptime, exposure, version, verbose=verbose)
Holger Niemann's avatar
Holger Niemann committed
353
    else:
354
#    except:
355
        print("download_NUC_by_program: cannot find the program")
356
        return False, 0, 0
Holger Niemann's avatar
Holger Niemann committed
357

358
def download_NUC_by_times(port, starttime, stoptime, exposure, testmode=False, version=0, verbose=0):
359
    '''
360
    download the NUC data for a given port, start and endtime in ns and exposure time of the camera
361
362
    INPUT
    ------
363
364
365
366
367
368
369
370
371
372
373
374
375
376
        port: integer or string
            port number of the AEF port of the camera
        starttime: integer
            time in ns where the search should start
        stoptime: integer
            time in sn where the search should end
        exposure: integer
            exposure time in microseconds
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive
        verbose: integer, optional    
            feedback level (details of print messages) 
377
378
    RESULT
    ------
379
380
381
382
383
384
        exist: boolean
            True if NUC data was found, False if not
        NUC: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        NUC_describtion: list
            a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
385
386
    NOTE
    ------
387
        function need to be adapted for OP2, if AEK ports come into opperation
388
    '''
389
    OP = IR_tools.get_OP_by_time(time_ns=starttime)
390
    if testmode:
391
        larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
392
    else:
393
        larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
Holger Niemann's avatar
Holger Niemann committed
394
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
395
396
    if version == 0:
        version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM", t_from=starttime, testmode=testmode)
Holger Niemann's avatar
Holger Niemann committed
397
398
399
400
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
401
        goon = True
Holger Niemann's avatar
Holger Niemann committed
402
    except urllib.error.URLError as e:
403
        print('download_NUC_by_times: Error! ', e)
404
        goon = False
405
406
    except Exception as e:
        print('download_NUC_by_times: Error! ', e)
407
408
409
410
        goon = False
    if goon:
        n = 0
        nuctimes = [0]
Holger Niemann's avatar
Holger Niemann committed
411
        for NUCpar in signal_list['values']:
412
413
            if NUCpar['structure']['Texp'] == exposure:
                nuctimes = [signal_list['dimensions'][2*n], signal_list['dimensions'][2*n+1]]
Holger Niemann's avatar
Holger Niemann committed
414
#                gain_i=NUCpar['structure']['gain_index']
415
416
417
418
                offset_i = NUCpar['structure']['offset_index']
            n += 1
        if nuctimes[0] != 0:
            NUC = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
419
            if NUC[0]:
420
421
                images = np.vsplit(NUC[2], np.shape(NUC[2])[0]/offset_i)
                return True, images, ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
Holger Niemann's avatar
Holger Niemann committed
422
            else:
423
424
                if verbose > 0:
                    print("download_NUC_by_times: NUC image for requested exposure time not found")
Holger Niemann's avatar
Holger Niemann committed
425
426
                return False, 0, 0
        else:
427
428
429
            if verbose > 0:
                print("download_NUC_by_times: NUC image for requested exposure time not found")
            return False, 0, 0
Holger Niemann's avatar
Holger Niemann committed
430
    else:
431
        return False, 0, 0
432

433
def get_NUC_by_program(port, program, exposure, version=0, testmode=False, verbose=0):
434
    '''
435
436
437
    Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera
    in a given port and time interval. Depending on time and camera, reconstruct
    the cold frame and/or compute the gain, offset from the hot and cold frame.
438
439
    INPUT
    ------
440
441
442
443
444
445
446
447
448
449
450
451
        port: integer or string
            port number of the AEF port of the camera
        program: string
            program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016'
        exposure: integer
            exposure time in microseconds
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive
        verbose: integer, optional    
            feedback level (details of print messages) 
452
453
    RESULT
    ------
454
455
456
457
458
459
        exist: boolean
            True if NUC data was found, False if not
        NUC: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        NUC_describtion: list
            a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
460
461
    NOTE
    ------
462
        function need to be adapted for OP2, if AEK ports come into opperation
463
464
    '''
#    prog=get_program_from_PID(program)
465
466
467
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
468
469
470
471
472
#    if prog[0]:
#        starttime=prog[1]['trigger']['0'][0]
#        stoptime=prog[1]['trigger']['1'][0]
    exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
    if exist:
473
        return get_NUC_by_times(port, starttime, stoptime, exposure, version=version, testmode=testmode, verbose=verbose)
474
475
    else:
#    except:
476
        print("get_NUC_by_program: Warning! cannot find the trigger timestamps for program", program)
477
478
479
480
481
#        return False, 0, 0
        OP = IR_tools.get_OP_by_time(program_str=program)
        return get_NUC_by_times(port, starttime, stoptime, exposure, version=version, testmode=testmode, OP=OP, verbose=verbose)

def get_NUC_by_times(port, starttime, stoptime, t_exp, version=0, testmode=False, OP=None, verbose=0):
482
483
484
    """
    Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera
    in a given port and time interval. Depending on time and camera, reconstruct
485
    the cold frame and/or compute the gain, offset from the hot and cold frame.
486
487
    the result is operation phase dependent, due to changes in the aquisition and saving programs
    
488
489
    INPUT
    ------
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
        port: integer or string
            port number of the AEF port of the camera
        starttime: integer
            time in ns where the search should start
        stoptime: integer
            time in sn where the search should end
        exposure: integer
            exposure time in microseconds
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive
        OP: string, optional, default None
            name of operation phase
        verbose: integer, optional    
            feedback level (details of print messages) 
506
507
    RESULT
    ------
508
509
510
511
512
513
        exist: boolean
            True if NUC data was found, False if not
        NUC: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        NUC_describtion: list
            a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
514
515
    NOTE
    ------
516
        function need to be adapted for OP2, if AEK ports come into opperation
517
    """
518
    if OP is None: #get operation phase if not given
519
        OP = IR_tools.get_OP_by_time(time_ns=stoptime)
520
    if OP == "OP1.2a": # check for operation phase
521
522
523
524
        t1 = stoptime
        t0 = starttime
        prog = read_program(t1)
        program = prog[1]['id']
525
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):#camera 31 and 21 need after 20171110 new coldframes due to a open shutter during the recording of the coldframes
526
            if verbose > 0:
527
                print("get_NUC_by_times: rebuilding coldframe")
528
            #use any way the rebuilt coldframe.
529
            sT = get_sensor_temp_by_program(port, program, testmode=testmode)[2][0]
530
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
531
532
            gotit, hotcold, describtion = download_hot_cold_reference_by_times(port, t_exp, testmode=False)
            del describtion
533
            if gotit:
534
535
                coldref = hotcold[1]
                hotref = hotcold[0]
536
            else:
537
538
539
#                raise Exception("get_NUC_by_times: unable to download reference frames")
                logging.warning("get_NUC_by_times: unable to download reference frames")
                return False, 0, 0
540
            filestring = 'AEF' + str(port) + '_et' + str(int(t_exp))
541
542
543
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
544
            if verbose > 0:
Peter Drewelow's avatar
Peter Drewelow committed
545
                print('get_NUC_by_times: calculate gain, offset and bad pixels')
546
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref, verbose=verbose-1)
547
548
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
549
550
551
552
            badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1)
            gain_error = 0
            offset_error = 0
            return True, [gain, offset, cirebuild, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
553
        elif (port == 11) or (port == 21): # case of the swaped cameras
554
            if verbose > 0:
Peter Drewelow's avatar
Peter Drewelow committed
555
                print('get_NUC_by_times: downloading NUC')
556
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
557
            gotit, hotcold, describtion = download_hot_cold_reference_by_times(port, t_exp, testmode=False)
558
            if gotit:
559
560
                coldref = hotcold[1]
                hotref = hotcold[0]
561
            else:
562
563
564
#                raise Exception("get_NUC_by_times: unable to download reference frames")
                logging.warning("get_NUC_by_times: unable to download reference frames")
                return False, 0, 0
565
566
            NUC_DL = download_NUC_by_times(port, t0, t1, t_exp, version, testmode)
            if NUC_DL[0] == False:
567
568
569
#                raise Exception("get_NUC_by_times: NUC was not found")
                logging.warning("get_NUC_by_times: NUC was not found")
                return False, 0, 0
570
571
            else:
                ci = NUC_DL[1][2]
572
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref, verbose=verbose-1)
573
574
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
575
            badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1)
576
577
578
            gain_error = 0
            offset_error = 0
            return True, [gain, offset, ci, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
579
        else: #all other cases, the "normal" case
580
            if verbose > 0:
Peter Drewelow's avatar
Peter Drewelow committed
581
                print('get_NUC_by_times: downloading NUC')
582
583
584
585
586
587
            NUC_DL = download_NUC_by_times(port, starttime, stoptime, t_exp, testmode=testmode)
            #True, images, ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
            if NUC_DL[0]:
                badpixels = find_badpixels(port,NUC_DL[1][0],NUC_DL[1][1],niterations=10, tolerance=10, verbose=verbose-1)
                NUC_DL[1][3]=badpixels
            return NUC_DL
588
589
590
591
    elif OP == "OP1.2b":
        gain_error = 0
        offset_error = 0
        gotit, hotcold, describtion = download_hot_cold_reference_by_times(port, t_exp, testmode=False)
592
        if gotit:
593
594
            coldref = hotcold[1]
            hotref = hotcold[0]
595
        else:
596
597
598
#            raise Exception("get_NUC_by_times: unable to download reference frames")
            logging.warning("get_NUC_by_times: unable to download reference frames")
            return False, 0, 0
599
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
600
        try:
601
            exist, _, coldframes = download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, frametype=0, version=0, testmode=testmode, verbose=verbose-1)
602
        except:
603
            exist = False
604
        if exist:#okay it exist, average the cold frames
605
            cold = np.zeros(np.shape(coldframes[0]), dtype=np.uint64)
606
            for ele in coldframes:
607
608
                cold += ele
            cold = np.asarray(cold/len(coldframes), dtype=np.uint16)
609
        else:
610
611
            print('get_NUC_by_times: Warning! no cold frames found. will use reference cold frame...')
            cold = np.copy(coldref)
612
613
614
615
#            return False, [0], [0]
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref, verbose=verbose-1)
        badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1)
        return True, [gain, offset, cold, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
616
    else:
617
618
619
#        raise Exception("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP")
        logging.warning("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP")
        return False, 0, 0
620

621
def download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, frametype=0, version=0, testmode=False, verbose=0):
622
    """
623
    download of the calibration raw files which are recorded between t0 and t1 trigger
624
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
625
    
626
627
    INPUT
    ------
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
        port: integer 
            port number of the AEF port of the camera
        t_exp: integer
            exposure time in microseconds
        starttime: integer
            time in ns where the search should start        
        stoptime: integer
            time in sn where the search should end 
        frametype: integer, default 0
            defines which type of frames should be returned, 0 for cold frames, 1 for background frames
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive        
        verbose: integer, optional    
            feedback level (details of print messages) 
644
645
    RESULT
    ------
646
647
648
649
650
651
        exist: boolean
            True if data was found, False if not
        time: list
            a list containing the time stamp of each frame
        frames: list
            a list of frames, 2D numpy arrays
652
653
    NOTE
    ------
654
        function need to be adapted for OP2, if AEK ports come into opperation
655
    """
656
657
    gotit, time_t, texp_t = get_exposure_by_times(port, int(starttime-100), int(stoptime-20e6), testmode=testmode)
    OP = IR_tools.get_OP_by_time(time_ns=stoptime)
658
    if gotit:
659
660
        expinds = np.where(texp_t == t_exp)[0]
        if len(expinds) == 0:
661
            logging.warning("download_calibration_raw_files_by_time: Error! cannot find the exposure time in the given data")
662
            return False, [0], [0]
663
    else:
664
        logging.warning("download_calibration_raw_files_by_time: Error! exposure time not found")
665
666
        return False, [0], [0]
    gotitf, timef, values_f = get_frametype_by_times(port, int(starttime-100), int(stoptime-20e6), testmode=testmode)
667
    if gotitf:
668
669
670
        typiinds = np.where(values_f[expinds] == frametype)[0]
        if len(typiinds) > 0:
            ref_t = [np.min(timef[expinds][typiinds]), np.max(timef[expinds][typiinds])]
Holger Niemann's avatar
Holger Niemann committed
671
#            print((ref_t[1]-ref_t[0])/1e9)
672
#            print(len(timef), len(timef[expinds]), len(timef[expinds][typiinds]))
673
        else:#okay the early data stuff or strange stuff
674
            if verbose > 0:
675
                print("download_calibration_raw_files_by_time: frame type was not identified assuming that the first part is the cold and the second one the background")
676
677
678
679
680
            frametimes = time_t[expinds]
            diftime = [frametimes[i]-frametimes[i-1] for i in range(1, len(frametimes))]
            turnpoint = np.where(np.asarray(diftime) > 11e6)[0][0]
            if frametype == 0:
                ref_t = [np.min(frametimes[0:turnpoint+1]), np.max(frametimes[0:turnpoint+1])]
681
#                print((ref_t[1]-ref_t[0])/1e9)
682
683
684
            elif frametype == 1:
#                print(len(frametimes[turnpoint+1:]), len(frametimes[0:turnpoint+1]), len(frametimes))
                ref_t = [np.min(frametimes[turnpoint+1:]), np.max(frametimes[turnpoint+1:])]
685
#                print((ref_t[1]-ref_t[0])/1e9)
686
            else:
687
688
689
#                raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
                logging.warning("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
                return False, [0], [0]
690
    else:
691
        print("download_calibration_raw_files_by_time: Error! frametype not found")
692
693
694
695
696
        frametimes = time_t[expinds]
        diftime = [frametimes[i]-frametimes[i-1] for i in range(1, len(frametimes))]
        turnpoint = np.where(np.asarray(diftime) > 11e6)[0][0]
        if frametype == 0:
            ref_t = [np.min(frametimes[0:turnpoint+1]), np.max(frametimes[0:turnpoint+1])]
697
            print((ref_t[1]-ref_t[0])/1e9)
698
699
700
        elif frametype == 1:
#                print(len(frametimes[turnpoint+1:]), len(frametimes[0:turnpoint+1]), len(frametimes))
            ref_t = [np.min(frametimes[turnpoint+1:]), np.max(frametimes[turnpoint+1:])]
701
702
            print((ref_t[1]-ref_t[0])/1e9)
        else:
703
704
705
#            raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
            logging.warning("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
            return False, [0], [0]
706
707
708
709
710
711
712
#        return False, [0], [0]
    t1date = datetime.datetime.utcfromtimestamp((stoptime-100)/1e9)
    t1date = t1date.isoformat()
    t0date = datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
    t0date = t0date.isoformat()
    if version == 0:
        version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM", t_from=starttime, t_to=stoptime, testmode=testmode)
713
    if testmode:
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
        larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
    else:
        larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
    timest0 = AKF_2.get_time_intervals(larchivepath, t0date.replace("T", " "), t1date.replace("T", " "))
    t_offset = timest0[-1][0]-time_t[0]
    if t_offset > 10e6:
        if verbose > 0:
            print("download_calibration_raw_files_by_time: time offset detected, try to correct this, offset is", t_offset)
    else:
        t_offset = 0
#            print("starttime frames:", np.min(timest0), "starttime metachannels:", time_t[0], "offset", t_offset)
    return download_raw_images_by_times(port, ref_t[0]+t_offset, ref_t[1]+t_offset, testmode=testmode, verbose=verbose-1)

def download_hot_cold_reference_by_times(port, exposure, starttime=1503907200000000000,
                                         testmode=False, version=0):
729
730
731
    """
    Loads the most recent hot and cold calibration frames for a starttime.
    Uses first calibration frames if time is not defined.
732
733
    INPUT
    ------
734
735
736
737
738
739
740
741
742
743
744
745
        port: integer or string
            port number of the AEF port of the camera, or string containing the full port name
        exposure: integer
            exposure time in microseconds
        starttime: integer
            time in ns where the search should start
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive        
        verbose: integer, optional    
            feedback level (details of print messages) 
746
747
    RESULT
    ------
748
749
750
751
752
753
        exist: boolean
            True if data was found, False if not
        images: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        image_describtion: list
            a list which entry in NUC is what, looks like ['hot','cold']
754
755
    NOTE
    ------
756
        function need to be adapted for OP2, if AEK ports come into opperation
757
    """
758
759
760
    OP = IR_tools.get_OP_by_time(time_ns=starttime)
    if isinstance(port, int):
        port = "AEF"+str(port)
761
    if testmode:
762
        larchivepath = testarchivepath+project+"/"+portpathdict[OP][port]+"raw_"
763
    else:
764
        larchivepath = archivepath+project+"/"+portpathdict[OP][port]+"raw_"
765
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
766
767
    if version == 0:
        version = get_latest_version(portpathdict[OP][port]+"raw_DATASTREAM", t_from=starttime)
768
    try:
769
770
        path_string = larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9))
        res = urllib.request.urlopen(path_string)
771
772
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
773
        goon = True
774
    except urllib.error.URLError as e:
775
776
        print('download_hot_cold_reference_by_times: Error! ', e)
        goon = False
777
    except Exception as e:
778
779
780
781
782
        print('download_hot_cold_reference_by_times: Error! ', e)
        goon = False
    if goon:
        COLDtime = 0
        HOTtime = 0
783
        try:
784
            COLDtime = signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
785
        except:
786
            print("download_hot_cold_reference_by_times: cold image for requested exposure time not found")
787
        try:
788
            HOTtime = signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
789
        except:
790
            print("download_hot_cold_reference_by_times: hot image for requested exposure time not found")
791
792
793
        images = []
        if HOTtime != 0:
            HOT = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
794
795
            if HOT[0]:
                images.append(HOT[2])
796
797
        if COLDtime != 0:
            COLD = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
798
799
            if COLD[0]:
                images.append(COLD[2])
800
801
802

        if HOT[0] and COLD[0] and len(images) == 2:
            return True, images, ['hot', 'cold']
803
        else:
804
            print("download_hot_cold_reference_by_times: Error! hot and cold image for requested exposure time not found")
805
            return False, 0, 0
806
    else:
807
808
809
        return False, 0, 0

def download_background_by_program(port, program, exposure, camera_filter=0, version=0, testmode=False, verbose=0):
810
    '''
811
    download the background from the AEF cameras in OP1.2a, which are recorded between t0 and t1 trigger
812
    returned exist, time, frame
813
814
    INPUT
    ------
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
        port: integer 
            port number of the AEF port of the camera        
        program: string
            program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016'
        exposure: integer
            exposure time in microseconds        
        camera_filter: integer or string, only needed for port 50
            filter number of the Infratec camera in port AEF50
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive        
        verbose: integer, optional    
            feedback level (details of print messages) 
		OP: string, optional, default None
            name of operation phase
831
832
    RESULT
    ------
833
834
835
836
837
838
        exist: boolean
            True if data was found, False if not
        time: list
            a list containing the time stamp of each frame
        frames: list
            a list of frames, 2D numpy arrays
839
840
    NOTE
    ------
841
        function need to be adapted for OP2, if AEK ports come into opperation
842
    '''
843
#    prog=get_program_from_PID(program)
844
845
846
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
847
848
849
850
851
#    if prog[0]:
#        starttime=prog[1]['trigger']['0'][0]
#        stoptime=prog[1]['trigger']['1'][0]
    exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
    if exist:
852
        return download_background_by_times(port, starttime, stoptime, exposure, camera_filter=camera_filter, version=version)
853
    else:#except:
854
        print("download_background_by_program: Error! cannot find the program {0}".format(program))
855
856
857
858
        return False, 0, 0

def download_background_by_times(port, starttime, stoptime, exposure, camera_filter=0,
                                 version=0, testmode=False, verbose=0):
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
    """
    download the background frames for a given time intervall in nanoseconds, defined by start and stop time
    
    INPUT
    ------
        port: integer 
            port number of the AEF port of the camera
        starttime: integer
            time in ns where the search should start        
        stoptime: integer
            time in sn where the search should end 
        exposure: integer
            exposure time in microseconds
        camera_filter: integer or string, only needed for port 50
            filter number of the Infratec camera in port AEF50
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive        
        verbose: integer, optional    
            feedback level (details of print messages) 

    RESULT
    ------
        exist: boolean
            True if data was found, False if not
        time: list
            a list containing the time stamp of each frame
        frames: list
            a list of frames, 2D numpy arrays
    NOTE
    ------
        function need to be adapted for OP2, if AEK ports come into opperation
    """
893
894
895
    OP = IR_tools.get_OP_by_time(time_ns=starttime)
    if OP == "OP1.2a":
        stream = portpathdict[OP]["AEF"+str(port)]+"background_"
896
        if testmode:
897
            larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
898
        else:
899
900
901
            larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
        if version == 0:
            version = get_latest_version(stream+"DATASTREAM", t_from=starttime, testmode=testmode)
902
903
904
905
        try:
            res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
906
            goon = True
907
        except urllib.error.URLError as e:
908
909
            print('download_background_by_times: Error! ', e)
            goon = False
910
        except Exception as e:
911
912
            print('download_background_by_times: Error! ', e)
            goon = False
913
        if goon:
914
915
            n = 0
            backtimes = [0]
916
            for backpar in signal_list['values']:
917
918
919
                if backpar['structure']['Texp'] == exposure:
                    if port == 50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
                        if backpar['structure']['filter'] == camera_filter:
920
                            backtimes = [signal_list['dimensions'][2*n], signal_list['dimensions'][2*n+1]]
921
                    else:
922
                        backtimes = [signal_list['dimensions'][2*n], signal_list['dimensions'][2*n+1]]
923
924
925
                n += 1
            if backtimes[0] != 0:
                backdat = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
926
                if backdat[0]:
927
                    return backdat#[True, backdat[2]]
Holger Niemann's avatar
Holger Niemann committed
928
                else:
929
                    print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
930
                    return False, 0, 0
Holger Niemann's avatar
Holger Niemann committed
931
            else:
932
                print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
933
                return False, 0, 0
Holger Niemann's avatar
Holger Niemann committed
934
        else:
935
936
            return False, 0, 0
    elif OP == "OP1.2b":
937
        NUC_DL = get_NUC_by_times(port, starttime, stoptime, t_exp=exposure, version=version, testmode=testmode, verbose=verbose-1)#download_NUC_by_times(port, t0, t1, t_exp, version)
938
        if NUC_DL[0] == False:
939
940
941
#            raise Warning("download_background_by_times: NUC was not found")
            logging.warning("download_background_by_times: NUC was not found")
            return False, 0, 0
942
943
944
945
        else:
            # extract NUC elements
            gain = []
            offset = []
946
947
            gain = np.array(NUC_DL[1][0])
            offset = np.array(NUC_DL[1][1])
948
            # check quality of NUC elements
949