downloadversionIRdata.py 92.5 KB
Newer Older
Holger Niemann's avatar
Holger Niemann committed
1
2
3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
4
updated on Tue Aug 21 10:20:00 2018
Holger Niemann's avatar
Holger Niemann committed
5

6
7
Version: 2.5.3
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2) . #of updates to add functionalities . #number of updates for bug fixes )
Holger Niemann's avatar
Holger Niemann committed
8
9
10
11
@author: holn
"""

import numpy as np
12
import IR_image_tools as IR_tools
13
from IR_config_constants import archivepath,portcamdict,camlimdict,valid_FOV_circle,valid_background_rectangle,TC_port,TC_channel,IRCamColdframes_fittingpath,exJet,exJet_trans
14
15
16
17
try:
    import W7Xrest.read_restdb as AKF_1
except:
    import read_restdb as AKF_1
Holger Niemann's avatar
Holger Niemann committed
18
19
20
import datetime
import urllib
import json
21
22
from PIL import Image
from io import BytesIO
23
from os.path import join
24
25
26
import matplotlib.pyplot as plt
try:        
    import multiprocessing    
27
    import archivedb as AKF_2
28
29
30
31
    fastDL=True
except Exception as E:
    print(E)
    fastDL=False
Holger Niemann's avatar
Holger Niemann committed
32

33

34
35
36
37
38
39
40
try:
    config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\v1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
    FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0]   
    del FOV
except:
    config_path=""

41
42
43
44
45
46
47
48
49
50
51
52
53
54
def get_OP_by_program(program):
    try:    
        bla=int(program.split(".")[0])
        time_ns=int(TimeToNs([bla // 10000,bla % 10000 // 100,bla % 10000 % 100],[10,0,0,0]))
        return get_OP_by_time(time_ns)
    except Exception as E:
        print(E)
        return None
    
def get_OP_by_time(time_ns):
    dateOP=datetime.datetime.utcfromtimestamp(time_ns/1e9)
    if dateOP.year==2017:
        if dateOP.month>8 and dateOP.month<12:
            return "OP1.2a"
55
        elif dateOP.month==8 and dateOP.day>=28:
56
57
58
59
60
61
62
63
64
65
66
67
68
            return "OP1.2a"
        elif dateOP.month==12 and dateOP.day<8:
            return "OP1.2a"
        else:
            return None        
    elif dateOP.year==2018:
        return "OP1.2b"
    elif dateOP.year<=2016 and dateOP.year>=2015:
        if (dateOP.year==2016 and dateOP.month<=3) or (dateOP.year==2015 and dateOP.month==12):
            return "OP1.1"
        else:
            return None

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
def get_latest_version(stream):
     """Find out latest version of given stream
     :param stream   url stream of interest
     :return int of version number of None if non-versioned or non-existing stream
     author: G. Schlisio, edit by holn
     """
     request =urllib.request.Request(archivepath + stream + "/_versions.json", headers={"Accept": "application/json"})
     try:
         response = urllib.request.urlopen(request)
         d = json.loads(response.read().decode('utf-8'))
     except urllib.error.HTTPError as ex:
         msg = ex.read()
         raise RuntimeError(msg)
     # detect unversioned or non-existing stream
     if d["versionInfo"] == []:
         return None
     versions = []
     for i in d['versionInfo']:
         versions.append(i['number'])

     return max(versions)

Holger Niemann's avatar
Holger Niemann committed
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
def TimeToNs(date,time):
    """
    TimeToNs(date,time)
    (list)  date    [year,month,day]
    (list)  time    [hours,minutes,seconds,microseconds]
    """ 
    date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3])
    div=date_time-datetime.datetime(1970,1,1,0,0,0)
    nsdate=div.total_seconds()*1e9
    return nsdate

def read_program(timestampstart,timestamp_end=0,tol=60):
    """
    read_program()
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
    if timestamp_end!=0:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestamp_end+tol*1e9))
    else:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestampstart+tol*1e9))
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw=res.read()
        res.close()
    except urllib.error.URLError as e:
        print('!get_program: Error opening URL')
        print(e)
        return False,0
    else:
        prog_string=prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl=prog_list['programs'][0]
        return True, pl

def read_restdb_old(request_url):
    """
    Reads JSON data from W7-X REST API
    Returns: 
        valid: access ok
        t: numpy-array of time
        signal: numpy-array of requested signals
132
    by H. Thomsen
Holger Niemann's avatar
Holger Niemann committed
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
    """
    try:
        res = urllib.request.urlopen(request_url)
    except urllib.error.URLError as e:
        print(e)
        return False, 0, -1
    else:
        signal_raw=res.read()
        res.close()
        signal_string=signal_raw.decode(encoding='UTF-8')
        signal_list = json.loads(signal_string)
        signal0=np.array(signal_list['values'])
        t=np.array(signal_list['dimensions'])
        return True, t, signal0

148
def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
Holger Niemann's avatar
Holger Niemann committed
149
150
151
    """
    download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
        time in ns
152
    Have to swap 11, 21 until correction in the database
Holger Niemann's avatar
Holger Niemann committed
153
    """    
154
    OP=IR_tools.get_OP_by_time(time_ns=time)
155
156
157
158
    if port == 21:
        port = 11
    elif port == 11:
        port = 21
159
    if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
160
161
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
        larchivepath=archivepath+stream#"QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
Holger Niemann's avatar
Holger Niemann committed
162
        query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
163
    elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
164
165
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_LUT_"
        larchivepath=archivepath+stream
Holger Niemann's avatar
Holger Niemann committed
166
167
168
169
        query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
        print("camera unknown, stopping here")
        raise Exception
170
171
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
172
173
    if verbose>0:
        print("LUT V"+str(version)+" is used")
Holger Niemann's avatar
Holger Niemann committed
174
    #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
175
    LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
176
177
    if LUTpar[0]:
        LUTid=LUTpar[2][0]['structure'][query]
178
        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
179
        if LUTs[0]:
180
            LUTs=LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
Holger Niemann's avatar
Holger Niemann committed
181
182
183
184
            LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True,LUT
        else:
185
            print("Warning: unable to download the LUTs")
Holger Niemann's avatar
Holger Niemann committed
186
187
188
189
            del LUTpar, LUTs
            return False,0
    else:
        del LUTpar
190
        print("Warning: unable to find LUTs, check your request")
Holger Niemann's avatar
Holger Niemann committed
191
192
        return False,0

193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215

def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
    
    Read LUT V3 from local files for testing new calibration.
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
    OP = get_OP_by_time(this_time)
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
        with open(LUT_dir+"\\"+filename) as data_file:  
            jfile=json.load(data_file)
        LUT=np.array(jfile['LUT']).swapaxes(0,1)
        if verbose>0:
            print("succesfully loaded V3 LUT from local directory")
        return True, LUT
    except Exception as E:
        print("ERROR in loading V3 LUTs",E)
        return False, []
    
216
def download_NUC_by_program(port,program,exposure,version=0):
Holger Niemann's avatar
Holger Niemann committed
217
    prog=AKF_1.get_program_from_PID(program)
218
219
220
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
Holger Niemann's avatar
Holger Niemann committed
221
222
223
224
225
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_NUC_by_times(port,starttime,stoptime,exposure,version)
    else:
226
#    except:
Holger Niemann's avatar
Holger Niemann committed
227
228
229
        print("cannot find the program")
        return False,0,0

230
def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
231

Holger Niemann's avatar
Holger Niemann committed
232
233
    larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_NUC_"
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
234
235
    if version==0:
        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_NUC_DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        
        n=0
        nuctimes=[0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp']==exposure:
                nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i=NUCpar['structure']['offset_index']                
            n+=1 
        if nuctimes[0]!=0:
            NUC=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
            if NUC[0]:
                images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)                    
                return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
            else:
                print("NUC image for requested exposure time not found")
                return False, 0, 0
        else:
            print("NUC image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
266

267
def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
268
269
270
271
272
273
274
    prog=AKF_1.get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
275
        return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose-1)
276
277
278
279
280
    else:
#    except:
        print("cannot find the program")
        return False,0,0
    
281
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
282
283
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if OP=="OP1.2a":
284
285
286
287
        t1=endtime
        t0=starttime
        prog=read_program(t1)
        program=prog[1]['id']
288
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):
289
            
290
291
292
293
294
295
296
297
298
299
300
301
302
303
            print("rebuilding coldframe")
            #use any way the rebuilt coldframe.
            sT = get_sensor_temp_by_program(port, program)[2][0]
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("unable to download reference frames")
            filestring = 'AEF' + str(port) + '_et' + str(t_exp)
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
304
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
            gain_error=0
            offset_error=0        
            return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        elif (port == 11) or (port == 21):
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("unable to download reference frames")
            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
            if NUC_DL[0]==False:
                raise Exception("NUC was not found")
            else:
                ci = NUC_DL[1][2]
324
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1)
325
326
327
328
329
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)                                
            gain_error=0
            offset_error=0
330
            return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
        else:
            return download_NUC_by_times(port,starttime,endtime,t_exp)
    elif OP=="OP1.2b":
        gain_error=0
        offset_error=0
        gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
        if gotit:
            coldref=hotcold[1]
            hotref=hotcold[0]
        else:
            raise Exception("unable to download reference frames")
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
        exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
        if exist:
            cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
            for ele in coldframes:
                cold+=ele                
            cold=np.asarray(cold/len(coldframes),dtype=np.uint16)        
        else:
            return False,[0],[0]
351
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
352
353
354
355
356
357
358
359
360
361
        badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
        return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
    else:
        raise Exception("unknown Operation phase or NUC method not implemented for this OP")

def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0):
    """
    download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
    """
Holger Niemann's avatar
Holger Niemann committed
362
    gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(endtime-20e6))
363
364
365
366
367
368
369
370
    if gotit:
        expinds=np.where(texp_t==t_exp)[0]
        if len(expinds)==0:
            print("cannot find the exposure time in the given data")
            return False,[0],[0]                               
    else:
        print("exposure time not found")
        return False,[0],[0]
Holger Niemann's avatar
Holger Niemann committed
371
    gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(endtime-20e6))
372
    if gotitf:
Holger Niemann's avatar
Holger Niemann committed
373
        typiinds=np.where(values_f[expinds]==frametype)[0]
374
        if len(typiinds)>0:
Holger Niemann's avatar
Holger Niemann committed
375
376
377
            ref_t=[np.min(timef[expinds][typiinds]),np.max(timef[expinds][typiinds])]
#            print((ref_t[1]-ref_t[0])/1e9)
#            print(len(timef),len(timef[expinds]),len(timef[expinds][typiinds]))
378
379
380
381
382
383
384
        else:#okay the early data stuff or strange stuff
            print("frame type was not identified assuming that the first part is the cold and the second one the background")
            frametimes=time_t[expinds]
            diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
            turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
            if frametype==0:
                ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
Holger Niemann's avatar
Holger Niemann committed
385
                print((ref_t[1]-ref_t[0])/1e9)
386
387
388
            elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
                ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
Holger Niemann's avatar
Holger Niemann committed
389
                print((ref_t[1]-ref_t[0])/1e9)
390
391
392
393
394
395
396
            else:
                raise Exception("requested Frametype unknown and not implemented!")
    else:
        print("frametype not found")
        return False,[0],[0]
    t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
    t1date=t1date.isoformat()
Holger Niemann's avatar
Holger Niemann committed
397
    t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
398
399
400
401
402
    t0date=t0date.isoformat()
    if version==0:
        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_raw_DATASTREAM")
    larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
    timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
Holger Niemann's avatar
Holger Niemann committed
403
    t_offset=timest0[-1][0]-time_t[0]
404
    if t_offset>10e6:
Holger Niemann's avatar
Holger Niemann committed
405
        print("time offset detected, try to correct this, offset is",t_offset)
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
    else:
        t_offset=0
#            print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
    return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset)
        
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
    larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_raw_"
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_raw_DATASTREAM")
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        

        COLDtime=0
        HOTtime=0
        try:
            COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
        except:
            print("cold image for requested exposure time not found")
        try:
            HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
        except:
            print("hot image for requested exposure time not found")
        images=[]
        if HOTtime!=0:
            HOT=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
            if HOT[0]:
                images.append(HOT[2])
        if COLDtime!=0:
            COLD=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
            if COLD[0]:
                images.append(COLD[2])
                
        if HOT[0] and COLD[0] and len(images)==2:
            return True,images,['hot','cold']
        else:
            print("hot and cold image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
Holger Niemann's avatar
Holger Niemann committed
453
    
454
def download_background_by_program(port,program,exposure,version=0):
455
456
457
    '''
    returned exist,time,frame
    '''
458
459
460
461
462
463
464
    prog=AKF_1.get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
465
        return download_background_by_times(port,starttime,stoptime,exposure,version)
466
    else:#except:
Holger Niemann's avatar
Holger Niemann committed
467
468
469
        print("cannot find the program")
        return False,0,0
    
470
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
471
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
472
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_background_" 
Holger Niemann's avatar
Holger Niemann committed
473
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_background_"      
474
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
475
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_background_"   
Holger Niemann's avatar
Holger Niemann committed
476
477
478
479
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_background_"   
    else:
        print("camera unknown, stopping here")
        raise Exception
480
481
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
482
483
484
485
486
487
488
489
490
491
492
493
494
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
        n=0
        backtimes=[0]
        for backpar in signal_list['values']:
            if backpar['structure']['Texp']==exposure:
495
                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
                    if backpar['structure']['filter']==camera_filter:
                        backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
                else:
                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
            n+=1
        if backtimes[0]!=0:
            backdat=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
            if backdat[0]:                                    
                return backdat#[True,backdat[2]]
            else:
                print("background image for requested exposure time(, filter) not found")
                return False,0,0
        else:
            print("background image for requested exposure time(, filter) not found")
            return False,0,0
    else:
        return False,0,0

514
def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False):
515
516
517
518
519
520
521
522
523
524
525
526
    "OP1.2b function"
    exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version)
    camera=portcamdict["OP1.2b"]["AEF"+str(port)]    
    background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
    for frame in backgroundframes:
        background+=frame
    background=np.asarray(background/len(backgroundframes),dtype=np.uint16)
    if plot_it:
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image unnuced")
    if not camera.split("_")[0]=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
527
        background=apply_NUC([background],gain,offset)[0]
528
    if plot_it:        
Holger Niemann's avatar
Holger Niemann committed
529
        
530
531
532
533
534
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image nuced")
    return exist,btime[0],background

535
def download_raw_images_by_program(port,program,time_s=0,version=0):
536
537
538
539
540
541
542
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
Holger Niemann's avatar
Holger Niemann committed
543
544
545
546
547
548
        if type(time_s)==list:
            tstart=time_s[0]
            tstop=time_s[1]
            if tstop<tstart:
                raise Exception("endtime before starttime")
            return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version)
Holger Niemann's avatar
Holger Niemann committed
549
        else:
Holger Niemann's avatar
Holger Niemann committed
550
551
552
553
            if time_s==0:
                return download_raw_images_by_times(port,starttime,stoptime,version)
            else:
                return download_raw_images_by_times(port,starttime,int(starttime+time_s*1e9),version)
554
    except:
Holger Niemann's avatar
Holger Niemann committed
555
556
557
        print("cannot find the program")
        return False,0,0

558
def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9):
559
560
561
562
563
564
565
566
567
568
569
570
571
572
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
    if OP=="OP1.2a":
        if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":        
            if version==0:
                version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
            larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
        elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
            if version==0:
                version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
            larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
        else:
            print("camera unknown, stopping here")
            raise Exception
    elif OP=="OP1.2b":
573
574
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
575
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
576
    
577
578
    if (stoptime-starttime)/intervalSize>1:        
        nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
Holger Niemann's avatar
Holger Niemann committed
579
        print("timewindow to large, splitting into smaller fractions ("+str(nrinterv)+")",(stoptime-starttime)/1e9)
Holger Niemann's avatar
Holger Niemann committed
580
        intervalls=[]
581
        for i in range(nrinterv):
582
            intervalls.append(int(starttime-10+i*intervalSize))
Holger Niemann's avatar
Holger Niemann committed
583
        intervalls.append(stoptime)
584
        for i in range(nrinterv):
Holger Niemann's avatar
Holger Niemann committed
585
            try:
Holger Niemann's avatar
Holger Niemann committed
586
                res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1]))
Holger Niemann's avatar
Holger Niemann committed
587
588
                signal_list = json.loads(res.read().decode('utf-8'))
                res.close()
589
590
591
                
                images=[np.array(ele, dtype=np.uint16) for ele in signal_list['values']]
#                signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
592
593
594
                t=np.array(signal_list['dimensions'])
                del signal_list
                if i==0:
595
                    allimages=images.copy()
Holger Niemann's avatar
Holger Niemann committed
596
597
                    time=t
                else:
598
                    allimages=allimages+images#np.append(ressignal,signal,axis=0)
Holger Niemann's avatar
Holger Niemann committed
599
                    time=np.append(time,t)
600
                del images,t
Holger Niemann's avatar
Holger Niemann committed
601
            except urllib.error.URLError as e:
Holger Niemann's avatar
Holger Niemann committed
602
                print(e)                
603
        return True,time,allimages
Holger Niemann's avatar
Holger Niemann committed
604
605
606
607
608
    else:
        try:
            res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
609
610
            images=[np.array(ele, dtype=np.uint16) for ele in signal_list['values']]
#            signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
611
612
            t=np.array(signal_list['dimensions'])
            del signal_list
613
            return True, t, images
Holger Niemann's avatar
Holger Niemann committed
614
615
616
        except urllib.error.URLError as e:
            print(e)
            return False, 0,-1
617

618
if fastDL:
619
    def download_raw_images_by_program_via_png(port,program,time_s=0,version=0,threads=1, verbose=0):
620
621
622
623
        prog=AKF_1.get_program_from_PID(program)
#        try:
#            t_program = AKF_2.get_program_from_to(program)
#            prog =AKF_2.get_program_list(t_program[0], t_program[1])
624
625
626
627
628
        if not prog[0]:
            if verbose>0:
                print("program not found, cannot download the data")
                return False,[0],[0]
        else:
629
630
            starttime=prog[1]['trigger']['1'][0]
            stoptime=prog[1]['trigger']['6'][0]
631
632
633
            success=True
            OP=get_OP_by_time(starttime)
            Cam=portcamdict[OP]['AEF'+str(port)]
634
635
636
637
638
639
640
641
642
643
644
645
            if OP=="OP1.2a":
                if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
                    if version==0:
                        version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
                    larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
                elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
                    if version==0:
                        version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
                    larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
                else:
                    raise Exception("Port number does not fit the known cameras")
            elif OP=="OP1.2b":
646
647
                if version==0:
                    version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
648
                larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
Holger Niemann's avatar
Holger Niemann committed
649
650
651
652
653
654
655
656
            if type(time_s)==list:
                tstart=time_s[0]
                tstop=time_s[1]
                if tstop<tstart:
                    raise Exception("endtime before starttime")
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
                stdate=stdate.isoformat()
                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
657
658
                enddate=enddate.isoformat()
            else:
Holger Niemann's avatar
Holger Niemann committed
659
660
661
662
663
664
665
666
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
                stdate=stdate.isoformat()
                if time_s==0:
                    enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
                    enddate=enddate.isoformat()
                else:
                    enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)  
                    enddate=enddate.isoformat()
667
                #"2017-11-15 08:00:00"
668
669
670
671
672
            try:
                times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
            except Exception as E:
                raise Warning(E)
                return False,[0],[0]            
673
674
675
676
677
678
679
680
681
            time=[]
            images=[]
            lnt=len(times)
            if threads==1:
                for i in range(lnt):
                    ele=times[lnt-1-i]
                    imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
                    if imag[0]:
                        time.append(ele[0])
682
                        images.append(np.array(imag[1],dtype=np.uint16))
683
684
                    else:
                        success=False
685
                return success,np.array(time),images
686
687
688
689
690
691
692
693
694
695
696
697
            else:
                tim=[]
                for i in range(lnt):
                    tim.append(times[lnt-1-i][0])
                intervalls=[]
                intervalSize=int(lnt/threads)
                for i in range(threads):
                    intervalls.append(int(i*intervalSize))
                intervalls.append(lnt)
                jobs = []
                out_q=multiprocessing.Queue()
                for i in range(threads):
698
699
                    if verbose>0:
                        print("Start Thread ",i+1)                
700
701
702
703
704
705
706
707
                    p = multiprocessing.Process(target=download_raw_images_png_by_times_thread, args=(port,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,))
                    jobs.append(p)
                    p.start()
                resultdict = []
                for i in range(threads):
                    resultdict.append(out_q.get())
                for p in jobs:
                    p.join()
708
709
                if verbose>0:
                    print("all threads are done")
710
711
712
713
714
715
                order=[]
                for ele in resultdict:
                    order.append(ele[0])
                    if len(np.where(np.asarray(ele[1])==False)[0])>0:
                        success=False
                
716
717
718
719
720
                times=[]#np.array(resultdict[order.index(0)][2])
                images=[]#np.array(resultdict[order.index(0)][3])
                for i in range(threads):
                    images=images+resultdict[order.index(i)][3]
                    times=times+resultdict[order.index(i)][2]
721
                del resultdict
722
                return success,np.array(times),images
723
724
#        except Exception as E:
#            raise Exception(E)
725
            
726
    
727
    def download_raw_images_png_by_times_thread(port,times,out_q,threadnumber,version=0):
728
729
730
731
        images=[]
        time=[]
        successes=[]
        for i in times:
732
            imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
733
            if imag[0]:
734
                images.append(np.array(imag[1],dtype=np.uint16))
735
#                print(np.min(imag[1]),np.max(imag[1]))
736
737
738
739
740
741
                time.append(i)
                successes.append(True)
            else:
                successes.append(False)
        out_q.put([threadnumber,successes,time,images])        
    #    return success,np.array(time),np.array(images,dtype=np.uint16)
742
743
744
745
746
747
748
749
750
751
752
753
754
755
        
    def convert_raw_to_temp_thread(port,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
        images=[]
        time=[]
        successes=[]
        for i in times:
            imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
            if imag[0]:
                images.append(np.array(imag[1],dtype=np.uint16))
                time.append(i)
                successes.append(True)
            else:
                successes.append(False)
        if give_ERROR:
756
            sucess,images,error_images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
757
        else:
758
            success,images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
759
760
761
762
        
        for i in range(len(images)):
            images[i]=(images[i]*FOV).astype(np.float32)
        print(datetime.datetime.now(),"correcting bad pixels")
763
        images=IR_tools.correct_images(images,badpixels)
764
        out_q.put([threadnumber,successes,time,images]) 
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
    
    def NUC_raw_thread(port,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
        images=[]
        time=[]
        successes=[]
        for i in times:
            imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
            if imag[0]:
                images.append(np.array(imag[1],dtype=np.uint16))
                time.append(i)
                successes.append(True)
            else:
                successes.append(False)
        if give_ERROR:
            images,error_images=apply_NUC(images,gain,offset,gain_error,offset_error,give_ERROR)
        else:
            images=apply_NUC(images,gain,offset,gain_error,offset_error,give_ERROR)
        
        for i in range(len(images)):
            images[i]=(images[i]*FOV).astype(np.float32)
        print(datetime.datetime.now(),"correcting bad pixels")
        images=IR_tools.correct_images(images,badpixels)
        out_q.put([threadnumber,successes,time,images]) 
788
        
789
def download_last_raw_image_by_time(port,starttime,stoptime,version=0):
790
791
    OP=get_OP_by_time(starttime)
    Cam=portcamdict[OP]['AEF'+str(port)]
792
    if Cam.split("_")[0]=="Infratec" and OP=="OP1.2a":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
793
794
        if version==0:
            version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
795
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
796
    elif Cam.split("_")[0]=="IRCam"or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
797
798
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
799
800
801
802
803
804
805
806
807
808
809
810
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
    else:
        raise Exception("camera unknown, stopping here")
    try:
        res = urllib.request.urlopen(larchivepath+"/_signal.png?from="+str(starttime-10)+"&upto="+str(stoptime))
        img = Image.open(BytesIO(res.read()))
        res.close()
#        pixelarray = np.array(img.getdata()).reshape(img.size[1], img.size[0])
        pixelarray = np.array(img,dtype=np.uint16)#.swapaxes(0,1)
        return True, pixelarray
    except urllib.error.URLError as e:
        print(e)
811
        return False, [-1]
Holger Niemann's avatar
Holger Niemann committed
812
    
813
def download_raw_parlog_by_program(port,program,version=0):
814
815
816
817
818
819
820
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
821
        return download_raw_parlog_by_times(port,starttime,stoptime,version)    
822
    except:
Holger Niemann's avatar
Holger Niemann committed
823
824
825
        print("cannot find the program")
        return False,0,0
    
826
def download_raw_parlog_by_times(port,starttime,stoptime,version=0):
827
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
828
829
        if version==0:
            version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG")
Holger Niemann's avatar
Holger Niemann committed
830
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)
831
    elif port in [10,11,20,21,30,31,40,41,51]:#elif camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
832
833
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG")
Holger Niemann's avatar
Holger Niemann committed
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)   
    else:
        print("camera unknown, stopping here")
        raise Exception
    try:
        res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
        return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']
    

850
def get_INFRATEC_filter_by_program(program,version=0):
851
852
853
854
855
856
857
    prog=AKF_1.get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
Holger Niemann's avatar
Holger Niemann committed
858
        return  get_INFRATEC_filter_by_times(starttime,stoptime,50,version)
859
    else:#except:
Holger Niemann's avatar
Holger Niemann committed
860
861
862
        print("cannot find the program")
        return False,0,0

863
def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0):
864
865
866
867
868
869
870
871
872
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
    if OP=="OP1.2a":
        if version==0:
            version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG")
        larchivepath=archivepath+"QRT_INFRATEC/AEF"+str(port)+"_raw_PARLOG/V"+str(version)
    elif OP=="OP1.2b":
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG")
        larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_raw_PARLOG/V"+str(version)
Holger Niemann's avatar
Holger Niemann committed
873
874
875
876
877
878
879
880
881
    try:
        res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
Holger Niemann's avatar
Holger Niemann committed
882
883
884
        return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']['filter']  
    else:
        return False, 0,-1
Holger Niemann's avatar
Holger Niemann committed
885
    
886
def get_exposure_by_program(port,program,version=0):
Holger Niemann's avatar
Holger Niemann committed
887
    prog=AKF_1.get_program_from_PID(program)
888
889
890
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
Holger Niemann's avatar
Holger Niemann committed
891
892
893
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
894
        return get_exposure_by_times(port,starttime,stoptime,version)
Holger Niemann's avatar
Holger Niemann committed
895
    else:
896
897
#    except Exception as E:
        print("cannot find the program, no exposure time available;")#, E)
Holger Niemann's avatar
Holger Niemann committed
898
899
        return False,0,0
        
900
def get_exposure_by_times(port,starttime,stoptime,version=0):
901
902
903
    OP=IR_tools.get_OP_by_time(time_ns=stoptime)
    if OP=="OP1.2a" and port==50:
#    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
904
905
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"   
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_" 
906
907
#    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
    else:
908
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_" 
909
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
910
    if version==0:
911
        version=get_latest_version(stream+"DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
912
913
    return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/3/exposuretime"+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
      
914
def get_camera_temp_by_program(port,program,version=0):
915
916
917
918
919
920
921
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
922
        return get_camera_temp_by_times(port,starttime,stoptime,version)
923
    except:
Holger Niemann's avatar
Holger Niemann committed
924
925
926
        print("cannot find the program")
        return False,0,0
        
927
def get_camera_temp_by_times(port,starttime,stoptime,version=0):
928
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
929
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"  
Holger Niemann's avatar
Holger Niemann committed
930
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
931
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
932
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
Holger Niemann's avatar
Holger Niemann committed
933
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
934
935
936
    else:
        print("Port unknown")
        raise Exception
937
938
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
939
940
941
942
943
    dummy=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if dummy[0]:
        return dummy
    else:
        return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratempetarure/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
Holger Niemann's avatar
Holger Niemann committed
944
    
945
def get_sensor_temp_by_program(port,program,version=0):
946
947
948
949
950
951
952
    prog=AKF_1.get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
953
        return get_sensor_temp_by_times(port,starttime,stoptime,version)
954
    else:#except:
Holger Niemann's avatar
Holger Niemann committed
955
956
957
        print("cannot find the program")
        return False,0,0
        
958
def get_sensor_temp_by_times(port,starttime,stoptime,version=0):
959
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
960
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
Holger Niemann's avatar
Holger Niemann committed
961
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
962
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
963
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
Holger Niemann's avatar
Holger Niemann committed
964
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
965
966
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
967
968
    return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))

969
def get_camera_framerate_by_program(port,program,version=0):
970
971
972
973
974
975
976
    prog=AKF_1.get_program_from_PID(program)
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
977
        return get_camera_framerate_by_times(port,starttime,stoptime,version)
978
979
#    except:
    else:
Holger Niemann's avatar
Holger Niemann committed
980
981
982
        print("cannot find the program")
        return False,0,0
        
983
def get_camera_framerate_by_times(port,starttime,stoptime,version=0):
984
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
985
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
Holger Niemann's avatar
Holger Niemann committed
986
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
987
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
988
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
Holger Niemann's avatar
Holger Niemann committed
989
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
990
991
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")    
Holger Niemann's avatar
Holger Niemann committed
992
993
994
    return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    

995
996
997
998
999
1000
def get_frametype_by_program(port,program,version=0):
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
For faster browsing, not all history is shown. View entire blame