downloadversionIRdata.py 307 KB
Newer Older
Holger Niemann's avatar
Holger Niemann committed
1
2
3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
4
updated on Tue Aug 21 10:20:00 2018
5
last update on Fr Nov 23 15:37:00 2018
Holger Niemann's avatar
Holger Niemann committed
6

7
Version: 3.5.0
8
9
10
11
12
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3)
            .
            #of updates to add functionalities
            .
            #number of updates for bug fixes )
Holger Niemann's avatar
Holger Niemann committed
13
14
@author: holn
"""
15
version = "V3.5.0"
Holger Niemann's avatar
Holger Niemann committed
16
17

import numpy as np
18
import IR_image_tools as IR_tools
19
from IR_da_config_constants import archivepath, testarchivepath, portcamdict, camlimdict, \
20
21
22
23
                                valid_FOV_circle, valid_background_rectangle, \
                                TC_port, TC_channel, IRCamColdframes_fittingpath, \
                                exJet, portpathdict, project, project_ana, \
                                heatflux_requestlist_path#, exJet_trans
24
25
26
27
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
Holger Niemann's avatar
Holger Niemann committed
28
import datetime
29
30
import urllib.request
import urllib.error
31
import logging
Holger Niemann's avatar
Holger Niemann committed
32
import json
33
34
from PIL import Image
from io import BytesIO
35
from os.path import join
36
import matplotlib.pyplot as plt
37
try:
38
    import threading
39
    import w7xarchive as AKF_2
40
41
42
    # disbale deprecation warnings, because of timezone warning spam
    import warnings
    warnings.filterwarnings("ignore", category=DeprecationWarning)
43
    fastDL = True
44
45
except Exception as E:
    print(E)
46
    fastDL = False
Holger Niemann's avatar
Holger Niemann committed
47

48

49
try:
50
51
52
    config_path = "\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\"\
    "Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
    FOV = plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:, :, 0]
53
54
    del FOV
except:
55
    config_path = ""
56

57

58
59
60
61
62
63
def get_started():
    """
    call it to get a short introduction to the available functions in this library\n
    it opens a weblink to a presentation and as well a flow chart
    """
    print("Welcome")
64
    link = "https://wikis.ipp-hgw.mpg.de/PhysicsW7X/images/9/9f/2019-04-24_20190424_How_to_access_Infrared_Data.pdf"
65
    path=str(repr(__file__)).split("downloadversionIRdata")[0].split("'")[1]
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
    image = path + "IR_data_access_flow_chart.png"
    try:
        import webbrowser
        webbrowser.open(link, new=2)
        try:
            os.startfile(image)
        except AttributeError:
            # Non-windows os:
            webbrowser.open(image)
    except Exception:
        # When ever that fails, just print the links:
        print("Some slides can be found in the wiki:")
        print(link)
        plt.figure()
        im=plt.imread(image)
        plt.imshow(im)        
        print("And a flow-chart for getting and using the data can be found here:")
        print(image)
        plt.show()

86
87
    

88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
def get_latest_version(stream, project="W7X", testmode=False, t_from=None,
                       t_to=None, program=None, verbose=0):
    """Find out latest version of given stream

     INPUT
     ------
         stream: string
             url stream of interest
         project: string, optional, default 'W7X'
             the main tree in the database
         testmode: boolean, optional, default False
             whether ArchiveDB oder Test Archive should be used for the request
         t_from: uint64, optional
             timestamp in nanosecond from where the data version should be identified
         t_to: uint64, optional
             timestamp in nanosecond up to where the data version should be identified
         program: string, optional
             program id as a string in the form of "yyyymmdd.pid"
         verbose: integer, optional, default=0
             defines the output level, 1 only this function, 2 this and one level below functions etc.
     RESULT
     ------
         version: integer or None
             return int of version number of None if non-versioned or non-existing stream
112
113
     author: G. Schlisio, edit by holn
     """
114
    if testmode:#check which archive should be used
115
116
117
118
        base = testarchivepath
    else:
        base = archivepath

119
    if program == None and t_from == None and t_to == None:# check wheather any time information was given
120
121
122
        request = urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})
    else:
        if program is not None:
123
124
125
126
#             prog=get_program_from_PID(program)
#             if prog[0]:
#                 t_from=prog[1]['trigger']['0'][0]
#                 t_to=prog[1]['trigger']['6'][0]
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
            exist, t_from, _, t_to = get_trigger_from_PID(program, None, testmode, verbose=verbose-1)
            if not exist:
                raise Exception("Cannot identify the Program")

        elif t_from is not None and t_to == None:
            t_to = int(t_from+100e9)
        elif t_from == None and t_to is not None:
            t_from = int(t_to-10e9)
        request = urllib.request.Request(base +project+"/" + stream + "/_versions.json?from="+str(t_from)+"&upto="+str(t_to), headers={"Accept": "application/json"})
    try:
        response = urllib.request.urlopen(request)
        d = json.loads(response.read().decode('utf-8'))
    except urllib.error.HTTPError as ex:
        msg = ex.read()
        raise RuntimeError(msg)
    except Exception as E:
        raise RuntimeError(E)
    else:
        response.close()
146
     # detect unversioned or non-existing stream
147
148
149
150
151
    if d["versionInfo"] == []:
        return None
    versions = []
    for i in d['versionInfo']:
        versions.append(i['number'])
152

153
    return max(versions)
154

155
def TimeToNs(date, time):
Holger Niemann's avatar
Holger Niemann committed
156
    """
157
158
159
160
161
162
163
164
165
166
167
168
    TimeToNs(date, time) 
    
    INPUT
    ------
         data: list
             [year, month, day]
         time: list
             [hours, minutes, seconds, microseconds]
    RESULT
    ------
         nsdate: integer
             nanosecond since 1.1.1970 0:00:00
169
170
171
172
173
    """
    date_time = datetime.datetime(date[0], date[1], date[2], time[0], time[1],
                                  time[2], time[3])
    div = date_time-datetime.datetime(1970, 1, 1, 0, 0, 0)
    nsdate = div.total_seconds()*1e9
174
    return int(nsdate)
Holger Niemann's avatar
Holger Niemann committed
175

176
def read_program(timestamp_start, timestamp_end=0, tol=60):
Holger Niemann's avatar
Holger Niemann committed
177
    """
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
    read_program() downloads the information from the W7-X ArchiveDB and provides the programnumber and triggers
    
    INPUT
    ------
        timestamp_start: integer
            start time in nanoseconds
        timestamp_end: inter, optional, default is 0
            time to stop the search in nanoseconds
        tol: integer, optional, default is 60
            toleranz in seconds around the timestamp_start for searching the program in the W7-X database
    RESULT
    ------
        exist: boolean
            True if data was found, False if not
        program: dictionary
            W7-X program information
    
Holger Niemann's avatar
Holger Niemann committed
195
196
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
197
    if timestamp_end == 0:
198
199
200
201
        timestamp_end = timestamp_start
    jsonurl = '{0}={1}&upto={2}'.format(program_url,
                                        int(timestamp_start-tol*1e9),
                                        int(timestamp_end+tol*1e9))
Holger Niemann's avatar
Holger Niemann committed
202
203
    try:
        res = urllib.request.urlopen(jsonurl)
204
        prog_raw = res.read()
Holger Niemann's avatar
Holger Niemann committed
205
206
        res.close()
    except urllib.error.URLError as e:
207
        print('read_program: Error opening URL')
Holger Niemann's avatar
Holger Niemann committed
208
        print(e)
209
        return False, 0
210
    except Exception as e:
211
212
        print('read_program: Error opening URL', e)
        return False, 0
Holger Niemann's avatar
Holger Niemann committed
213
    else:
214
        prog_string = prog_raw.decode(encoding='UTF-8')
Holger Niemann's avatar
Holger Niemann committed
215
        prog_list = json.loads(prog_string)
216
        pl = prog_list['programs'][0]
Holger Niemann's avatar
Holger Niemann committed
217
218
        return True, pl

219

Holger Niemann's avatar
Holger Niemann committed
220

221
222
def download_LUT(port, time, exposure=0, emissivity=0, camera_filter=0, version=0,
                 testmode=False, verbose=0):
Holger Niemann's avatar
Holger Niemann committed
223
    """
224
    download_LUT(port, time, exposure=0, emissivity=0, camera_filter=0, version=1):
225
        download of the look up table for the infrared cameras from the database for OP1.2(a+b)
226
    Have to swap 11, 21 until correction in the database
227
228
    INPUT
    ------
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
        port: interger
            number of the AEF camera port
        time: integer
            nanosecond timestamp for the LUT            
        exposure: integer
            requested exposure time in microseconds
        emissivity: float, string or integer
            emissivity value to identify which LUT is requested
        camera_filter: inter or string, only needed for port 50
            filter number of the Infratec camera in port AEF50
        version: integer, optional, default 0
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional, default False
            if True, the data will be loaded from the test archive
        verbose: integer, optional, default 0
            feedback level (details of print messages)     
245
246
    RESULT
    ------
247
248
249
250
        exist: boolean
            True if Look up table was found, False if not
        LUT: list
            contains the calibration information: [digital level, temperature, temperature error]
251
252
    NOTE
    ------
253
        function need to be adapted for OP2, if AEK ports come into opperation
254
    """
255
256
    OP = IR_tools.get_OP_by_time(time_ns=time)#getting operation phase
    if port == 21:# swapping both ports (21 and 11)
257
258
259
        port = 11
    elif port == 11:
        port = 21
260
    if testmode: #check which archive should be used
261
        larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
262
    else:
263
        larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
264
    #camera dependend request string generation
265
266
267
268
    if port == 50 and OP == "OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
        query = "Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    elif port in [10, 11, 20, 21, 30, 31, 40, 41, 51] or OP == "OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
        query = "Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
Holger Niemann's avatar
Holger Niemann committed
269
    else:
270
271
272
273
274
#        print("download_LUT: Error! Camera unknown, stopping here.")
        logging.warning("download_LUT: Error! Camera unknown, stopping here.")
        return False, 0
#        raise Exception
    if version == 0:#version check, if 0
275
        version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_PARLOG", t_from=time, testmode=testmode)
276
    if verbose > 0:
277
        print("download_LUT: LUT V"+str(version)+" is used")
278
279
    #time=int(fu.TimeToNs([2017,9,26], [8, 0, 0, 0]))
    LUTpar = read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
280
    if LUTpar[0]:
281
        LUTid = LUTpar[2][0]['structure'][query]
282
283
284
        #getting here now the time from the parlog
        Timetamp = LUTpar[2][0]['structure']['timestamp']
        LUTs = read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(Timetamp-10)+"&upto="+str(Timetamp+20))
Holger Niemann's avatar
Holger Niemann committed
285
        if LUTs[0]:
286
287
            LUTs = LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
            LUT = [LUTs[0], LUTs[LUTid], LUTs[LUTid+1]]
Holger Niemann's avatar
Holger Niemann committed
288
            del LUTpar, LUTs
289
            return True, LUT
Holger Niemann's avatar
Holger Niemann committed
290
        else:
291
            print("download_LUT: Warning! unable to download the LUTs")
Holger Niemann's avatar
Holger Niemann committed
292
            del LUTpar, LUTs
293
            return False, 0
Holger Niemann's avatar
Holger Niemann committed
294
295
    else:
        del LUTpar
296
        print("download_LUT: Warning! unable to find LUTs, check your request")
297
298
        return False, 0

Holger Niemann's avatar
Holger Niemann committed
299

300

301
def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, verbose=0):#emissivity_steel=0.31,
302
303
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
304

305
    Read LUT V3 from local files for testing new calibration.
306
307
    INPUT
    ------
308
309
310
311
312
313
314
315
316
317
318
319
        port: integer or string
            the port number of the AEF port        
        this_time: integer
            nanosecond timestamp for the LUT            
        t_exp: integer
            requested exposure time in microseconds
        emissivity: float, string or integer
            emissivity value to identify which LUT is requested
        cfilter: inter or string, only needed for port 50
            filter number of the Infratec camera in port AEF50
        verbose: integer, optional, default 0 
            feedback level (details of print messages) 
320
321
    RESULT
    ------
322
323
324
325
        exist: boolean
            True if Look up table was found, False if not
        LUT: list
            contains the calibration information: [digital level, temperature, temperature error]
326
327
    NOTE
    ------
328
        function need to be adapted for OP2, if AEK ports come into opperation
329
330
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
331
    OP = IR_tools.get_OP_by_time(time_ns=this_time)
332
333
334
335
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
336
337
338
339
        with open(LUT_dir+"\\"+filename) as data_file:
            jfile = json.load(data_file)
        LUT = np.array(jfile['LUT']).swapaxes(0, 1)
        if verbose > 0:
340
            print("read_LUT_from_file: succesfully loaded V3 LUT from local directory")
341
342
        return True, LUT
    except Exception as E:
343
        print("read_LUT_from_file: ERROR in loading V3 LUTs", E)
344
        return False, []
345
346

def download_NUC_by_program(port, program, exposure, version=0, testmode=False, verbose=0):
347
    '''
348
    download the NUC data for a given port, program and exposure time of the camera
349
350
    INPUT
    ------
351
352
353
354
355
356
357
358
359
360
361
362
        port: integer or string
            port number of the AEF port of the camera
        program: string
            program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016'
        exposure: integer
            exposure time in microseconds
        version: integer, optional, default 0
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional, default False
            if True, the data will be loaded from the test archive
        verbose: integer, optional, default 0
            feedback level (details of print messages) 
363
364
    RESULT
    ------
365
366
367
368
369
370
        exist: boolean
            True if NUC data was found, False if not
        NUC: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        NUC_describtion: list
            a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
371
372
    NOTE
    ------
373
        function need to be adapted for OP2, if AEK ports come into opperation
374
375
    '''
#    prog=get_program_from_PID(program)
376
377
378
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
379
380
381
382
383
#    if prog[0]:
#        starttime=prog[1]['trigger']['0'][0]
#        stoptime=prog[1]['trigger']['1'][0]
    exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
    if exist:
384
        return download_NUC_by_times(port, starttime, stoptime, exposure, version, verbose=verbose)
Holger Niemann's avatar
Holger Niemann committed
385
    else:
386
#    except:
387
        print("download_NUC_by_program: cannot find the program")
388
        return False, 0, 0
Holger Niemann's avatar
Holger Niemann committed
389

390
def download_NUC_by_times(port, starttime, stoptime, exposure, testmode=False, version=0, verbose=0):
391
    '''
392
    download the NUC data for a given port, start and endtime in ns and exposure time of the camera
393
394
    INPUT
    ------
395
396
397
398
399
400
401
402
403
404
405
406
407
408
        port: integer or string
            port number of the AEF port of the camera
        starttime: integer
            time in ns where the search should start
        stoptime: integer
            time in sn where the search should end
        exposure: integer
            exposure time in microseconds
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive
        verbose: integer, optional    
            feedback level (details of print messages) 
409
410
    RESULT
    ------
411
412
413
414
415
416
        exist: boolean
            True if NUC data was found, False if not
        NUC: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        NUC_describtion: list
            a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
417
418
    NOTE
    ------
419
        function need to be adapted for OP2, if AEK ports come into opperation
420
    '''
421
    OP = IR_tools.get_OP_by_time(time_ns=starttime)
422
    if testmode:
423
        larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
424
    else:
425
        larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
Holger Niemann's avatar
Holger Niemann committed
426
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
427
428
    if version == 0:
        version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM", t_from=starttime, testmode=testmode)
Holger Niemann's avatar
Holger Niemann committed
429
430
431
432
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
433
        goon = True
Holger Niemann's avatar
Holger Niemann committed
434
    except urllib.error.URLError as e:
435
        print('download_NUC_by_times: Error! ', e)
436
        goon = False
437
438
    except Exception as e:
        print('download_NUC_by_times: Error! ', e)
439
440
441
442
        goon = False
    if goon:
        n = 0
        nuctimes = [0]
Holger Niemann's avatar
Holger Niemann committed
443
        for NUCpar in signal_list['values']:
444
445
            if NUCpar['structure']['Texp'] == exposure:
                nuctimes = [signal_list['dimensions'][2*n], signal_list['dimensions'][2*n+1]]
Holger Niemann's avatar
Holger Niemann committed
446
#                gain_i=NUCpar['structure']['gain_index']
447
448
449
450
                offset_i = NUCpar['structure']['offset_index']
            n += 1
        if nuctimes[0] != 0:
            NUC = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
451
            if NUC[0]:
452
453
                images = np.vsplit(NUC[2], np.shape(NUC[2])[0]/offset_i)
                return True, images, ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
Holger Niemann's avatar
Holger Niemann committed
454
            else:
455
456
                if verbose > 0:
                    print("download_NUC_by_times: NUC image for requested exposure time not found")
Holger Niemann's avatar
Holger Niemann committed
457
458
                return False, 0, 0
        else:
459
460
461
            if verbose > 0:
                print("download_NUC_by_times: NUC image for requested exposure time not found")
            return False, 0, 0
Holger Niemann's avatar
Holger Niemann committed
462
    else:
463
        return False, 0, 0
464

465
def get_NUC_by_program(port, program, exposure, version=0, testmode=False, verbose=0):
466
    '''
467
468
469
    Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera
    in a given port and time interval. Depending on time and camera, reconstruct
    the cold frame and/or compute the gain, offset from the hot and cold frame.
470
471
    INPUT
    ------
472
473
474
475
476
477
478
479
480
481
482
483
        port: integer or string
            port number of the AEF port of the camera
        program: string
            program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016'
        exposure: integer
            exposure time in microseconds
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive
        verbose: integer, optional    
            feedback level (details of print messages) 
484
485
    RESULT
    ------
486
487
488
489
490
491
        exist: boolean
            True if NUC data was found, False if not
        NUC: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        NUC_describtion: list
            a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
492
493
    NOTE
    ------
494
        function need to be adapted for OP2, if AEK ports come into opperation
495
496
    '''
#    prog=get_program_from_PID(program)
497
498
499
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
500
501
502
503
504
#    if prog[0]:
#        starttime=prog[1]['trigger']['0'][0]
#        stoptime=prog[1]['trigger']['1'][0]
    exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
    if exist:
505
        return get_NUC_by_times(port, starttime, stoptime, exposure, version=version, testmode=testmode, verbose=verbose)
506
507
    else:
#    except:
508
        print("get_NUC_by_program: Warning! cannot find the trigger timestamps for program", program)
509
510
511
512
513
#        return False, 0, 0
        OP = IR_tools.get_OP_by_time(program_str=program)
        return get_NUC_by_times(port, starttime, stoptime, exposure, version=version, testmode=testmode, OP=OP, verbose=verbose)

def get_NUC_by_times(port, starttime, stoptime, t_exp, version=0, testmode=False, OP=None, verbose=0):
514
515
516
    """
    Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera
    in a given port and time interval. Depending on time and camera, reconstruct
517
    the cold frame and/or compute the gain, offset from the hot and cold frame.
518
519
    the result is operation phase dependent, due to changes in the aquisition and saving programs
    
520
521
    INPUT
    ------
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
        port: integer or string
            port number of the AEF port of the camera
        starttime: integer
            time in ns where the search should start
        stoptime: integer
            time in sn where the search should end
        exposure: integer
            exposure time in microseconds
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive
        OP: string, optional, default None
            name of operation phase
        verbose: integer, optional    
            feedback level (details of print messages) 
538
539
    RESULT
    ------
540
541
542
543
544
545
        exist: boolean
            True if NUC data was found, False if not
        NUC: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        NUC_describtion: list
            a list which entry in NUC is what, looks like ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
546
547
    NOTE
    ------
548
        function need to be adapted for OP2, if AEK ports come into opperation
549
    """
550
    if OP is None: #get operation phase if not given
551
        OP = IR_tools.get_OP_by_time(time_ns=stoptime)
552
    if OP == "OP1.2a": # check for operation phase
553
554
555
556
        t1 = stoptime
        t0 = starttime
        prog = read_program(t1)
        program = prog[1]['id']
557
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):#camera 31 and 21 need after 20171110 new coldframes due to a open shutter during the recording of the coldframes
558
            if verbose > 0:
559
                print("get_NUC_by_times: rebuilding coldframe")
560
            #use any way the rebuilt coldframe.
561
            sT = get_sensor_temp_by_program(port, program, testmode=testmode)[2][0]
562
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
563
564
            gotit, hotcold, describtion = download_hot_cold_reference_by_times(port, t_exp, testmode=False)
            del describtion
565
            if gotit:
566
567
                coldref = hotcold[1]
                hotref = hotcold[0]
568
            else:
569
570
571
#                raise Exception("get_NUC_by_times: unable to download reference frames")
                logging.warning("get_NUC_by_times: unable to download reference frames")
                return False, 0, 0
572
            filestring = 'AEF' + str(port) + '_et' + str(int(t_exp))
573
574
575
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
576
            if verbose > 0:
Peter Drewelow's avatar
Peter Drewelow committed
577
                print('get_NUC_by_times: calculate gain, offset and bad pixels')
578
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref, verbose=verbose-1)
579
580
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
581
582
583
584
            badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1)
            gain_error = 0
            offset_error = 0
            return True, [gain, offset, cirebuild, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
585
        elif (port == 11) or (port == 21): # case of the swaped cameras
586
            if verbose > 0:
Peter Drewelow's avatar
Peter Drewelow committed
587
                print('get_NUC_by_times: downloading NUC')
588
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
589
            gotit, hotcold, describtion = download_hot_cold_reference_by_times(port, t_exp, testmode=False)
590
            if gotit:
591
592
                coldref = hotcold[1]
                hotref = hotcold[0]
593
            else:
594
595
596
#                raise Exception("get_NUC_by_times: unable to download reference frames")
                logging.warning("get_NUC_by_times: unable to download reference frames")
                return False, 0, 0
597
598
            NUC_DL = download_NUC_by_times(port, t0, t1, t_exp, version, testmode)
            if NUC_DL[0] == False:
599
600
601
#                raise Exception("get_NUC_by_times: NUC was not found")
                logging.warning("get_NUC_by_times: NUC was not found")
                return False, 0, 0
602
603
            else:
                ci = NUC_DL[1][2]
604
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref, verbose=verbose-1)
605
606
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
607
            badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1)
608
609
610
            gain_error = 0
            offset_error = 0
            return True, [gain, offset, ci, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
611
        else: #all other cases, the "normal" case
612
            if verbose > 0:
Peter Drewelow's avatar
Peter Drewelow committed
613
                print('get_NUC_by_times: downloading NUC')
614
615
616
617
618
619
            NUC_DL = download_NUC_by_times(port, starttime, stoptime, t_exp, testmode=testmode)
            #True, images, ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
            if NUC_DL[0]:
                badpixels = find_badpixels(port,NUC_DL[1][0],NUC_DL[1][1],niterations=10, tolerance=10, verbose=verbose-1)
                NUC_DL[1][3]=badpixels
            return NUC_DL
620
621
622
623
    elif OP == "OP1.2b":
        gain_error = 0
        offset_error = 0
        gotit, hotcold, describtion = download_hot_cold_reference_by_times(port, t_exp, testmode=False)
624
        if gotit:
625
626
            coldref = hotcold[1]
            hotref = hotcold[0]
627
        else:
628
629
630
#            raise Exception("get_NUC_by_times: unable to download reference frames")
            logging.warning("get_NUC_by_times: unable to download reference frames")
            return False, 0, 0
631
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
632
        try:
633
            exist, _, coldframes = download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, frametype=0, version=0, testmode=testmode, verbose=verbose-1)
634
        except:
635
            exist = False
636
        if exist:#okay it exist, average the cold frames
637
            cold = np.zeros(np.shape(coldframes[0]), dtype=np.uint64)
638
            for ele in coldframes:
639
640
                cold += ele
            cold = np.asarray(cold/len(coldframes), dtype=np.uint16)
641
        else:
642
643
            print('get_NUC_by_times: Warning! no cold frames found. will use reference cold frame...')
            cold = np.copy(coldref)
644
645
646
647
#            return False, [0], [0]
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref, verbose=verbose-1)
        badpixels = find_badpixels(port, gain, offset, niterations=10, tolerance=10, verbose=verbose-1)
        return True, [gain, offset, cold, badpixels, gain_error, offset_error], ['gain', 'offset', 'cold', 'badpixels', 'gain_error', 'offset_error']
648
    else:
649
650
651
#        raise Exception("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP")
        logging.warning("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP")
        return False, 0, 0
652

653
def download_calibration_raw_files_by_time(port, t_exp, starttime, stoptime, frametype=0, version=0, testmode=False, verbose=0):
654
    """
655
    download of the calibration raw files which are recorded between t0 and t1 trigger
656
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
657
    
658
659
    INPUT
    ------
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
        port: integer 
            port number of the AEF port of the camera
        t_exp: integer
            exposure time in microseconds
        starttime: integer
            time in ns where the search should start        
        stoptime: integer
            time in sn where the search should end 
        frametype: integer, default 0
            defines which type of frames should be returned, 0 for cold frames, 1 for background frames
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive        
        verbose: integer, optional    
            feedback level (details of print messages) 
676
677
    RESULT
    ------
678
679
680
681
682
683
        exist: boolean
            True if data was found, False if not
        time: list
            a list containing the time stamp of each frame
        frames: list
            a list of frames, 2D numpy arrays
684
685
    NOTE
    ------
686
        function need to be adapted for OP2, if AEK ports come into opperation
687
    """
688
689
    gotit, time_t, texp_t = get_exposure_by_times(port, int(starttime-100), int(stoptime-20e6), testmode=testmode)
    OP = IR_tools.get_OP_by_time(time_ns=stoptime)
690
    if gotit:
691
692
        expinds = np.where(texp_t == t_exp)[0]
        if len(expinds) == 0:
693
            logging.warning("download_calibration_raw_files_by_time: Error! cannot find the exposure time in the given data")
694
            return False, [0], [0]
695
    else:
696
        logging.warning("download_calibration_raw_files_by_time: Error! exposure time not found")
697
698
        return False, [0], [0]
    gotitf, timef, values_f = get_frametype_by_times(port, int(starttime-100), int(stoptime-20e6), testmode=testmode)
699
    if gotitf:
700
701
702
        typiinds = np.where(values_f[expinds] == frametype)[0]
        if len(typiinds) > 0:
            ref_t = [np.min(timef[expinds][typiinds]), np.max(timef[expinds][typiinds])]
Holger Niemann's avatar
Holger Niemann committed
703
#            print((ref_t[1]-ref_t[0])/1e9)
704
#            print(len(timef), len(timef[expinds]), len(timef[expinds][typiinds]))
705
        else:#okay the early data stuff or strange stuff
706
            if verbose > 0:
707
                print("download_calibration_raw_files_by_time: frame type was not identified assuming that the first part is the cold and the second one the background")
708
709
710
711
712
            frametimes = time_t[expinds]
            diftime = [frametimes[i]-frametimes[i-1] for i in range(1, len(frametimes))]
            turnpoint = np.where(np.asarray(diftime) > 11e6)[0][0]
            if frametype == 0:
                ref_t = [np.min(frametimes[0:turnpoint+1]), np.max(frametimes[0:turnpoint+1])]
713
#                print((ref_t[1]-ref_t[0])/1e9)
714
715
716
            elif frametype == 1:
#                print(len(frametimes[turnpoint+1:]), len(frametimes[0:turnpoint+1]), len(frametimes))
                ref_t = [np.min(frametimes[turnpoint+1:]), np.max(frametimes[turnpoint+1:])]
717
#                print((ref_t[1]-ref_t[0])/1e9)
718
            else:
719
720
721
#                raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
                logging.warning("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
                return False, [0], [0]
722
    else:
723
        print("download_calibration_raw_files_by_time: Error! frametype not found")
724
725
726
727
728
        frametimes = time_t[expinds]
        diftime = [frametimes[i]-frametimes[i-1] for i in range(1, len(frametimes))]
        turnpoint = np.where(np.asarray(diftime) > 11e6)[0][0]
        if frametype == 0:
            ref_t = [np.min(frametimes[0:turnpoint+1]), np.max(frametimes[0:turnpoint+1])]
729
            print((ref_t[1]-ref_t[0])/1e9)
730
731
732
        elif frametype == 1:
#                print(len(frametimes[turnpoint+1:]), len(frametimes[0:turnpoint+1]), len(frametimes))
            ref_t = [np.min(frametimes[turnpoint+1:]), np.max(frametimes[turnpoint+1:])]
733
734
            print((ref_t[1]-ref_t[0])/1e9)
        else:
735
736
737
#            raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
            logging.warning("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
            return False, [0], [0]
738
739
740
741
742
743
744
#        return False, [0], [0]
    t1date = datetime.datetime.utcfromtimestamp((stoptime-100)/1e9)
    t1date = t1date.isoformat()
    t0date = datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
    t0date = t0date.isoformat()
    if version == 0:
        version = get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM", t_from=starttime, t_to=stoptime, testmode=testmode)
745
    if testmode:
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
        larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
    else:
        larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
    timest0 = AKF_2.get_time_intervals(larchivepath, t0date.replace("T", " "), t1date.replace("T", " "))
    t_offset = timest0[-1][0]-time_t[0]
    if t_offset > 10e6:
        if verbose > 0:
            print("download_calibration_raw_files_by_time: time offset detected, try to correct this, offset is", t_offset)
    else:
        t_offset = 0
#            print("starttime frames:", np.min(timest0), "starttime metachannels:", time_t[0], "offset", t_offset)
    return download_raw_images_by_times(port, ref_t[0]+t_offset, ref_t[1]+t_offset, testmode=testmode, verbose=verbose-1)

def download_hot_cold_reference_by_times(port, exposure, starttime=1503907200000000000,
                                         testmode=False, version=0):
761
762
763
    """
    Loads the most recent hot and cold calibration frames for a starttime.
    Uses first calibration frames if time is not defined.
764
765
    INPUT
    ------
766
767
768
769
770
771
772
773
774
775
776
777
        port: integer or string
            port number of the AEF port of the camera, or string containing the full port name
        exposure: integer
            exposure time in microseconds
        starttime: integer
            time in ns where the search should start
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive        
        verbose: integer, optional    
            feedback level (details of print messages) 
778
779
    RESULT
    ------
780
781
782
783
784
785
        exist: boolean
            True if data was found, False if not
        images: list
            a list vontaining images of gain, offset, cold, badpixels, gain_error and offset_error
        image_describtion: list
            a list which entry in NUC is what, looks like ['hot','cold']
786
787
    NOTE
    ------
788
        function need to be adapted for OP2, if AEK ports come into opperation
789
    """
790
791
792
    OP = IR_tools.get_OP_by_time(time_ns=starttime)
    if isinstance(port, int):
        port = "AEF"+str(port)
793
    if testmode:
794
        larchivepath = testarchivepath+project+"/"+portpathdict[OP][port]+"raw_"
795
    else:
796
        larchivepath = archivepath+project+"/"+portpathdict[OP][port]+"raw_"
797
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
798
799
    if version == 0:
        version = get_latest_version(portpathdict[OP][port]+"raw_DATASTREAM", t_from=starttime)
800
    try:
801
802
        path_string = larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9))
        res = urllib.request.urlopen(path_string)
803
804
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
805
        goon = True
806
    except urllib.error.URLError as e:
807
808
        print('download_hot_cold_reference_by_times: Error! ', e)
        goon = False
809
    except Exception as e:
810
811
812
813
814
        print('download_hot_cold_reference_by_times: Error! ', e)
        goon = False
    if goon:
        COLDtime = 0
        HOTtime = 0
815
        try:
816
            COLDtime = signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
817
        except:
818
            print("download_hot_cold_reference_by_times: cold image for requested exposure time not found")
819
        try:
820
            HOTtime = signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
821
        except:
822
            print("download_hot_cold_reference_by_times: hot image for requested exposure time not found")
823
824
825
        images = []
        if HOTtime != 0:
            HOT = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
826
827
            if HOT[0]:
                images.append(HOT[2])
828
829
        if COLDtime != 0:
            COLD = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
830
831
            if COLD[0]:
                images.append(COLD[2])
832
833
834

        if HOT[0] and COLD[0] and len(images) == 2:
            return True, images, ['hot', 'cold']
835
        else:
836
            print("download_hot_cold_reference_by_times: Error! hot and cold image for requested exposure time not found")
837
            return False, 0, 0
838
    else:
839
840
841
        return False, 0, 0

def download_background_by_program(port, program, exposure, camera_filter=0, version=0, testmode=False, verbose=0):
842
    '''
843
    download the background from the AEF cameras in OP1.2a, which are recorded between t0 and t1 trigger
844
    returned exist, time, frame
845
846
    INPUT
    ------
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
        port: integer 
            port number of the AEF port of the camera        
        program: string
            program id in the form of 'YYYYMMDD.XXX', e.g. '20181016.016'
        exposure: integer
            exposure time in microseconds        
        camera_filter: integer or string, only needed for port 50
            filter number of the Infratec camera in port AEF50
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive        
        verbose: integer, optional    
            feedback level (details of print messages) 
		OP: string, optional, default None
            name of operation phase
863
864
    RESULT
    ------
865
866
867
868
869
870
        exist: boolean
            True if data was found, False if not
        time: list
            a list containing the time stamp of each frame
        frames: list
            a list of frames, 2D numpy arrays
871
872
    NOTE
    ------
873
        function need to be adapted for OP2, if AEK ports come into opperation
874
    '''
875
#    prog=get_program_from_PID(program)
876
877
878
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
879
880
881
882
883
#    if prog[0]:
#        starttime=prog[1]['trigger']['0'][0]
#        stoptime=prog[1]['trigger']['1'][0]
    exist, starttime, stoptime, _ = get_trigger_from_PID(program, port, testmode, verbose=verbose-1)
    if exist:
884
        return download_background_by_times(port, starttime, stoptime, exposure, camera_filter=camera_filter, version=version)
885
    else:#except:
886
        print("download_background_by_program: Error! cannot find the program {0}".format(program))
887
888
889
890
        return False, 0, 0

def download_background_by_times(port, starttime, stoptime, exposure, camera_filter=0,
                                 version=0, testmode=False, verbose=0):
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
    """
    download the background frames for a given time intervall in nanoseconds, defined by start and stop time
    
    INPUT
    ------
        port: integer 
            port number of the AEF port of the camera
        starttime: integer
            time in ns where the search should start        
        stoptime: integer
            time in sn where the search should end 
        exposure: integer
            exposure time in microseconds
        camera_filter: integer or string, only needed for port 50
            filter number of the Infratec camera in port AEF50
        version: integer, optional
            Version of the data in the archiveDB, in case of 0 the highest version will be used
        testmode: boolean, optional
            if True, the data will be loaded from the test archive        
        verbose: integer, optional    
            feedback level (details of print messages) 

    RESULT
    ------
        exist: boolean
            True if data was found, False if not
        time: list
            a list containing the time stamp of each frame
        frames: list
            a list of frames, 2D numpy arrays
    NOTE
    ------
        function need to be adapted for OP2, if AEK ports come into opperation
    """
925
926
927
    OP = IR_tools.get_OP_by_time(time_ns=starttime)
    if OP == "OP1.2a":
        stream = portpathdict[OP]["AEF"+str(port)]+"background_"
928
        if testmode:
929
            larchivepath = testarchivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
930
        else:
931
932
933
            larchivepath = archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
        if version == 0:
            version = get_latest_version(stream+"DATASTREAM", t_from=starttime, testmode=testmode)
934
935
936
937
        try:
            res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
938
            goon = True
939
        except urllib.error.URLError as e:
940
941
            print('download_background_by_times: Error! ', e)
            goon = False
942
        except Exception as e:
943
944
            print('download_background_by_times: Error! ', e)
            goon = False
945
        if goon:
946
947
            n = 0
            backtimes = [0]
948
            for backpar in signal_list['values']:
949
950
951
                if backpar['structure']['Texp'] == exposure:
                    if port == 50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
                        if backpar['structure']['filter'] == camera_filter:
952
                            backtimes = [signal_list['dimensions'][2*n], signal_list['dimensions'][2*n+1]]
953
                    else:
954
                        backtimes = [signal_list['dimensions'][2*n], signal_list['dimensions'][2*n+1]]
955
956
957
                n += 1
            if backtimes[0] != 0:
                backdat = read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
958
                if backdat[0]:
959
                    return backdat#[True, backdat[2]]
Holger Niemann's avatar
Holger Niemann committed
960
                else:
961
                    print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
962
                    return