downloadversionIRdata.py 97 KB
Newer Older
Holger Niemann's avatar
Holger Niemann committed
1
2
3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
4
updated on Tue Aug 21 10:20:00 2018
Holger Niemann's avatar
Holger Niemann committed
5

6
Version: 2.6.1
7
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2) . #of updates to add functionalities . #number of updates for bug fixes )
Holger Niemann's avatar
Holger Niemann committed
8
9
10
11
@author: holn
"""

import numpy as np
12
import IR_image_tools as IR_tools
13
from IR_config_constants import archivepath,portcamdict,camlimdict,valid_FOV_circle,valid_background_rectangle,TC_port,TC_channel,IRCamColdframes_fittingpath,exJet,exJet_trans
14
15
16
17
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
Holger Niemann's avatar
Holger Niemann committed
18
19
20
import datetime
import urllib
import json
21
22
from PIL import Image
from io import BytesIO
23
from os.path import join
24
25
26
import matplotlib.pyplot as plt
try:        
    import multiprocessing    
27
    import archivedb as AKF_2
28
29
30
31
    fastDL=True
except Exception as E:
    print(E)
    fastDL=False
Holger Niemann's avatar
Holger Niemann committed
32

33

34
try:
35
    config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
36
37
38
39
40
    FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0]   
    del FOV
except:
    config_path=""

41
42
43
44
45
46
47
48
49
50
51
52
53
54
def get_OP_by_program(program):
    try:    
        bla=int(program.split(".")[0])
        time_ns=int(TimeToNs([bla // 10000,bla % 10000 // 100,bla % 10000 % 100],[10,0,0,0]))
        return get_OP_by_time(time_ns)
    except Exception as E:
        print(E)
        return None
    
def get_OP_by_time(time_ns):
    dateOP=datetime.datetime.utcfromtimestamp(time_ns/1e9)
    if dateOP.year==2017:
        if dateOP.month>8 and dateOP.month<12:
            return "OP1.2a"
55
        elif dateOP.month==8 and dateOP.day>=28:
56
57
58
59
60
61
62
63
64
65
66
67
68
            return "OP1.2a"
        elif dateOP.month==12 and dateOP.day<8:
            return "OP1.2a"
        else:
            return None        
    elif dateOP.year==2018:
        return "OP1.2b"
    elif dateOP.year<=2016 and dateOP.year>=2015:
        if (dateOP.year==2016 and dateOP.month<=3) or (dateOP.year==2015 and dateOP.month==12):
            return "OP1.1"
        else:
            return None

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
def get_latest_version(stream):
     """Find out latest version of given stream
     :param stream   url stream of interest
     :return int of version number of None if non-versioned or non-existing stream
     author: G. Schlisio, edit by holn
     """
     request =urllib.request.Request(archivepath + stream + "/_versions.json", headers={"Accept": "application/json"})
     try:
         response = urllib.request.urlopen(request)
         d = json.loads(response.read().decode('utf-8'))
     except urllib.error.HTTPError as ex:
         msg = ex.read()
         raise RuntimeError(msg)
     # detect unversioned or non-existing stream
     if d["versionInfo"] == []:
         return None
     versions = []
     for i in d['versionInfo']:
         versions.append(i['number'])

     return max(versions)

Holger Niemann's avatar
Holger Niemann committed
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
def TimeToNs(date,time):
    """
    TimeToNs(date,time)
    (list)  date    [year,month,day]
    (list)  time    [hours,minutes,seconds,microseconds]
    """ 
    date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3])
    div=date_time-datetime.datetime(1970,1,1,0,0,0)
    nsdate=div.total_seconds()*1e9
    return nsdate

def read_program(timestampstart,timestamp_end=0,tol=60):
    """
    read_program()
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
    if timestamp_end!=0:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestamp_end+tol*1e9))
    else:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestampstart+tol*1e9))
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw=res.read()
        res.close()
    except urllib.error.URLError as e:
        print('!get_program: Error opening URL')
        print(e)
        return False,0
    else:
        prog_string=prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl=prog_list['programs'][0]
        return True, pl

def read_restdb_old(request_url):
    """
    Reads JSON data from W7-X REST API
    Returns: 
        valid: access ok
        t: numpy-array of time
        signal: numpy-array of requested signals
132
    by H. Thomsen
Holger Niemann's avatar
Holger Niemann committed
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
    """
    try:
        res = urllib.request.urlopen(request_url)
    except urllib.error.URLError as e:
        print(e)
        return False, 0, -1
    else:
        signal_raw=res.read()
        res.close()
        signal_string=signal_raw.decode(encoding='UTF-8')
        signal_list = json.loads(signal_string)
        signal0=np.array(signal_list['values'])
        t=np.array(signal_list['dimensions'])
        return True, t, signal0

148
def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
Holger Niemann's avatar
Holger Niemann committed
149
150
151
    """
    download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
        time in ns
152
    Have to swap 11, 21 until correction in the database
Holger Niemann's avatar
Holger Niemann committed
153
    """    
154
    OP=IR_tools.get_OP_by_time(time_ns=time)
155
156
157
158
    if port == 21:
        port = 11
    elif port == 11:
        port = 21
159
    if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
160
161
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
        larchivepath=archivepath+stream#"QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
Holger Niemann's avatar
Holger Niemann committed
162
        query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
163
    elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
164
165
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_LUT_"
        larchivepath=archivepath+stream
Holger Niemann's avatar
Holger Niemann committed
166
167
168
169
        query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
        print("camera unknown, stopping here")
        raise Exception
170
171
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
172
173
    if verbose>0:
        print("LUT V"+str(version)+" is used")
Holger Niemann's avatar
Holger Niemann committed
174
    #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
175
    LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
176
177
    if LUTpar[0]:
        LUTid=LUTpar[2][0]['structure'][query]
178
        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
179
        if LUTs[0]:
180
            LUTs=LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
Holger Niemann's avatar
Holger Niemann committed
181
182
183
184
            LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True,LUT
        else:
185
            print("Warning: unable to download the LUTs")
Holger Niemann's avatar
Holger Niemann committed
186
187
188
189
            del LUTpar, LUTs
            return False,0
    else:
        del LUTpar
190
        print("Warning: unable to find LUTs, check your request")
Holger Niemann's avatar
Holger Niemann committed
191
192
        return False,0

193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215

def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
    
    Read LUT V3 from local files for testing new calibration.
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
    OP = get_OP_by_time(this_time)
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
        with open(LUT_dir+"\\"+filename) as data_file:  
            jfile=json.load(data_file)
        LUT=np.array(jfile['LUT']).swapaxes(0,1)
        if verbose>0:
            print("succesfully loaded V3 LUT from local directory")
        return True, LUT
    except Exception as E:
        print("ERROR in loading V3 LUTs",E)
        return False, []
    
216
def download_NUC_by_program(port,program,exposure,version=0):
217
    prog=get_program_from_PID(program)
218
219
220
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
Holger Niemann's avatar
Holger Niemann committed
221
222
223
224
225
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_NUC_by_times(port,starttime,stoptime,exposure,version)
    else:
226
#    except:
Holger Niemann's avatar
Holger Niemann committed
227
228
229
        print("cannot find the program")
        return False,0,0

230
def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
231

Holger Niemann's avatar
Holger Niemann committed
232
233
    larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_NUC_"
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
234
235
    if version==0:
        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_NUC_DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        
        n=0
        nuctimes=[0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp']==exposure:
                nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i=NUCpar['structure']['offset_index']                
            n+=1 
        if nuctimes[0]!=0:
254
            NUC=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
255
256
257
258
259
260
261
262
263
264
265
            if NUC[0]:
                images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)                    
                return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
            else:
                print("NUC image for requested exposure time not found")
                return False, 0, 0
        else:
            print("NUC image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
266

267
def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
268
    prog=get_program_from_PID(program)
269
270
271
272
273
274
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
275
        return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose-1)
276
277
278
279
280
    else:
#    except:
        print("cannot find the program")
        return False,0,0
    
281
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
282
283
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if OP=="OP1.2a":
284
285
286
287
        t1=endtime
        t0=starttime
        prog=read_program(t1)
        program=prog[1]['id']
288
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):
289
            
290
291
292
293
294
295
296
297
298
299
300
301
302
303
            print("rebuilding coldframe")
            #use any way the rebuilt coldframe.
            sT = get_sensor_temp_by_program(port, program)[2][0]
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("unable to download reference frames")
            filestring = 'AEF' + str(port) + '_et' + str(t_exp)
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
304
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
            gain_error=0
            offset_error=0        
            return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        elif (port == 11) or (port == 21):
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("unable to download reference frames")
            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
            if NUC_DL[0]==False:
                raise Exception("NUC was not found")
            else:
                ci = NUC_DL[1][2]
324
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1)
325
326
327
328
329
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)                                
            gain_error=0
            offset_error=0
330
            return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
        else:
            return download_NUC_by_times(port,starttime,endtime,t_exp)
    elif OP=="OP1.2b":
        gain_error=0
        offset_error=0
        gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
        if gotit:
            coldref=hotcold[1]
            hotref=hotcold[0]
        else:
            raise Exception("unable to download reference frames")
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
        exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
        if exist:
            cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
            for ele in coldframes:
                cold+=ele                
            cold=np.asarray(cold/len(coldframes),dtype=np.uint16)        
        else:
            return False,[0],[0]
351
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
352
353
354
355
356
357
358
359
360
361
        badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
        return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
    else:
        raise Exception("unknown Operation phase or NUC method not implemented for this OP")

def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0):
    """
    download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
    """
Holger Niemann's avatar
Holger Niemann committed
362
    gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(endtime-20e6))
363
364
365
366
367
368
369
370
    if gotit:
        expinds=np.where(texp_t==t_exp)[0]
        if len(expinds)==0:
            print("cannot find the exposure time in the given data")
            return False,[0],[0]                               
    else:
        print("exposure time not found")
        return False,[0],[0]
Holger Niemann's avatar
Holger Niemann committed
371
    gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(endtime-20e6))
372
    if gotitf:
Holger Niemann's avatar
Holger Niemann committed
373
        typiinds=np.where(values_f[expinds]==frametype)[0]
374
        if len(typiinds)>0:
Holger Niemann's avatar
Holger Niemann committed
375
376
377
            ref_t=[np.min(timef[expinds][typiinds]),np.max(timef[expinds][typiinds])]
#            print((ref_t[1]-ref_t[0])/1e9)
#            print(len(timef),len(timef[expinds]),len(timef[expinds][typiinds]))
378
379
380
381
382
383
384
        else:#okay the early data stuff or strange stuff
            print("frame type was not identified assuming that the first part is the cold and the second one the background")
            frametimes=time_t[expinds]
            diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
            turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
            if frametype==0:
                ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
Holger Niemann's avatar
Holger Niemann committed
385
                print((ref_t[1]-ref_t[0])/1e9)
386
387
388
            elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
                ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
Holger Niemann's avatar
Holger Niemann committed
389
                print((ref_t[1]-ref_t[0])/1e9)
390
391
392
393
394
395
396
            else:
                raise Exception("requested Frametype unknown and not implemented!")
    else:
        print("frametype not found")
        return False,[0],[0]
    t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
    t1date=t1date.isoformat()
Holger Niemann's avatar
Holger Niemann committed
397
    t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
398
399
400
401
402
    t0date=t0date.isoformat()
    if version==0:
        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_raw_DATASTREAM")
    larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
    timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
Holger Niemann's avatar
Holger Niemann committed
403
    t_offset=timest0[-1][0]-time_t[0]
404
    if t_offset>10e6:
Holger Niemann's avatar
Holger Niemann committed
405
        print("time offset detected, try to correct this, offset is",t_offset)
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
    else:
        t_offset=0
#            print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
    return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset)
        
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
    larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_raw_"
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_raw_DATASTREAM")
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        

        COLDtime=0
        HOTtime=0
        try:
            COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
        except:
            print("cold image for requested exposure time not found")
        try:
            HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
        except:
            print("hot image for requested exposure time not found")
        images=[]
        if HOTtime!=0:
438
            HOT=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
439
440
441
            if HOT[0]:
                images.append(HOT[2])
        if COLDtime!=0:
442
            COLD=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
443
444
445
446
447
448
449
450
451
452
            if COLD[0]:
                images.append(COLD[2])
                
        if HOT[0] and COLD[0] and len(images)==2:
            return True,images,['hot','cold']
        else:
            print("hot and cold image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
Holger Niemann's avatar
Holger Niemann committed
453
    
454
def download_background_by_program(port,program,exposure,version=0):
455
456
457
    '''
    returned exist,time,frame
    '''
458
    prog=get_program_from_PID(program)
459
460
461
462
463
464
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
465
        return download_background_by_times(port,starttime,stoptime,exposure,version)
466
    else:#except:
Holger Niemann's avatar
Holger Niemann committed
467
468
469
        print("cannot find the program")
        return False,0,0
    
470
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
471
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
472
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_background_" 
Holger Niemann's avatar
Holger Niemann committed
473
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_background_"      
474
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
475
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_background_"   
Holger Niemann's avatar
Holger Niemann committed
476
477
478
479
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_background_"   
    else:
        print("camera unknown, stopping here")
        raise Exception
480
481
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
482
483
484
485
486
487
488
489
490
491
492
493
494
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
        n=0
        backtimes=[0]
        for backpar in signal_list['values']:
            if backpar['structure']['Texp']==exposure:
495
                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
496
497
498
499
500
501
                    if backpar['structure']['filter']==camera_filter:
                        backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
                else:
                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
            n+=1
        if backtimes[0]!=0:
502
            backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
503
504
505
506
507
508
509
510
511
512
513
            if backdat[0]:                                    
                return backdat#[True,backdat[2]]
            else:
                print("background image for requested exposure time(, filter) not found")
                return False,0,0
        else:
            print("background image for requested exposure time(, filter) not found")
            return False,0,0
    else:
        return False,0,0

514
def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False):
515
516
517
518
519
520
521
522
523
524
525
526
    "OP1.2b function"
    exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version)
    camera=portcamdict["OP1.2b"]["AEF"+str(port)]    
    background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
    for frame in backgroundframes:
        background+=frame
    background=np.asarray(background/len(backgroundframes),dtype=np.uint16)
    if plot_it:
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image unnuced")
    if not camera.split("_")[0]=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
527
        background=apply_NUC([background],gain,offset)[0]
528
    if plot_it:        
Holger Niemann's avatar
Holger Niemann committed
529
        
530
531
532
533
534
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image nuced")
    return exist,btime[0],background

535
def download_raw_images_by_program(port,program,time_s=0,version=0,verbose=0):
536
537
538
539
540
541
542
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
Holger Niemann's avatar
Holger Niemann committed
543
544
545
546
547
        if type(time_s)==list:
            tstart=time_s[0]
            tstop=time_s[1]
            if tstop<tstart:
                raise Exception("endtime before starttime")
548
            return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
549
        else:
Holger Niemann's avatar
Holger Niemann committed
550
            if time_s==0:
551
                return download_raw_images_by_times(port,starttime,stoptime,version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
552
            else:
553
                return download_raw_images_by_times(port,starttime,int(starttime+time_s*1e9),version,verbose=verbose-1)
554
    except:
555
556
        if verbose>0:
            print("cannot find the program")
Holger Niemann's avatar
Holger Niemann committed
557
558
        return False,0,0

559
def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
560
561
562
563
564
565
566
567
568
569
570
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
    if OP=="OP1.2a":
        if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":        
            if version==0:
                version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
            larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
        elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
            if version==0:
                version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
            larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
        else:
571
572
            if verbose>0:
                print("camera unknown, stopping here")
573
574
            raise Exception
    elif OP=="OP1.2b":
575
576
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
577
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
578
    
579
580
    if (stoptime-starttime)/intervalSize>1:        
        nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
581
582
        if verbose>0:
            print("timewindow to large, splitting into smaller fractions ("+str(nrinterv)+")",(stoptime-starttime)/1e9)
Holger Niemann's avatar
Holger Niemann committed
583
        intervalls=[]
584
        for i in range(nrinterv):
585
            intervalls.append(int(starttime-10+i*intervalSize))
Holger Niemann's avatar
Holger Niemann committed
586
        intervalls.append(stoptime)
587
        for i in range(nrinterv):
Holger Niemann's avatar
Holger Niemann committed
588
            try:
Holger Niemann's avatar
Holger Niemann committed
589
                res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1]))
Holger Niemann's avatar
Holger Niemann committed
590
591
                signal_list = json.loads(res.read().decode('utf-8'))
                res.close()
592
593
594
                
                images=[np.array(ele, dtype=np.uint16) for ele in signal_list['values']]
#                signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
595
596
597
                t=np.array(signal_list['dimensions'])
                del signal_list
                if i==0:
598
                    allimages=images.copy()
Holger Niemann's avatar
Holger Niemann committed
599
600
                    time=t
                else:
601
                    allimages=allimages+images#np.append(ressignal,signal,axis=0)
Holger Niemann's avatar
Holger Niemann committed
602
                    time=np.append(time,t)
603
                del images,t
Holger Niemann's avatar
Holger Niemann committed
604
            except urllib.error.URLError as e:
605
606
                if verbose>0:
                    print(e)                
607
        return True,time,allimages
Holger Niemann's avatar
Holger Niemann committed
608
609
610
611
612
    else:
        try:
            res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
613
614
            images=[np.array(ele, dtype=np.uint16) for ele in signal_list['values']]
#            signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
615
616
            t=np.array(signal_list['dimensions'])
            del signal_list
617
            return True, t, images
Holger Niemann's avatar
Holger Niemann committed
618
        except urllib.error.URLError as e:
619
620
            if verbose>0:
                print(e)
Holger Niemann's avatar
Holger Niemann committed
621
            return False, 0,-1
622

623
if fastDL:
624
    def download_raw_images_by_program_via_png(port,program,time_s=0,version=0,threads=1, verbose=0):
625
        prog=get_program_from_PID(program)
626
627
628
#        try:
#            t_program = AKF_2.get_program_from_to(program)
#            prog =AKF_2.get_program_list(t_program[0], t_program[1])
629
630
631
632
633
        if not prog[0]:
            if verbose>0:
                print("program not found, cannot download the data")
                return False,[0],[0]
        else:
634
635
            starttime=prog[1]['trigger']['1'][0]
            stoptime=prog[1]['trigger']['6'][0]
636
637
638
            success=True
            OP=get_OP_by_time(starttime)
            Cam=portcamdict[OP]['AEF'+str(port)]
639
640
641
642
643
644
645
646
647
648
649
650
            if OP=="OP1.2a":
                if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
                    if version==0:
                        version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
                    larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
                elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
                    if version==0:
                        version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
                    larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
                else:
                    raise Exception("Port number does not fit the known cameras")
            elif OP=="OP1.2b":
651
652
                if version==0:
                    version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
653
                larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
Holger Niemann's avatar
Holger Niemann committed
654
655
656
657
658
659
660
661
            if type(time_s)==list:
                tstart=time_s[0]
                tstop=time_s[1]
                if tstop<tstart:
                    raise Exception("endtime before starttime")
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
                stdate=stdate.isoformat()
                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
662
663
                enddate=enddate.isoformat()
            else:
Holger Niemann's avatar
Holger Niemann committed
664
665
666
667
668
669
670
671
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
                stdate=stdate.isoformat()
                if time_s==0:
                    enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
                    enddate=enddate.isoformat()
                else:
                    enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)  
                    enddate=enddate.isoformat()
672
                #"2017-11-15 08:00:00"
673
674
675
676
677
            try:
                times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
            except Exception as E:
                raise Warning(E)
                return False,[0],[0]            
678
679
680
681
682
683
684
685
686
            time=[]
            images=[]
            lnt=len(times)
            if threads==1:
                for i in range(lnt):
                    ele=times[lnt-1-i]
                    imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
                    if imag[0]:
                        time.append(ele[0])
687
                        images.append(np.array(imag[1],dtype=np.uint16))
688
689
                    else:
                        success=False
690
                return success,np.array(time),images
691
692
693
694
695
696
697
698
699
700
701
702
            else:
                tim=[]
                for i in range(lnt):
                    tim.append(times[lnt-1-i][0])
                intervalls=[]
                intervalSize=int(lnt/threads)
                for i in range(threads):
                    intervalls.append(int(i*intervalSize))
                intervalls.append(lnt)
                jobs = []
                out_q=multiprocessing.Queue()
                for i in range(threads):
703
704
                    if verbose>0:
                        print("Start Thread ",i+1)                
705
706
707
708
709
710
711
712
                    p = multiprocessing.Process(target=download_raw_images_png_by_times_thread, args=(port,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,))
                    jobs.append(p)
                    p.start()
                resultdict = []
                for i in range(threads):
                    resultdict.append(out_q.get())
                for p in jobs:
                    p.join()
713
714
                if verbose>0:
                    print("all threads are done")
715
716
717
718
719
720
                order=[]
                for ele in resultdict:
                    order.append(ele[0])
                    if len(np.where(np.asarray(ele[1])==False)[0])>0:
                        success=False
                
721
722
723
724
725
                times=[]#np.array(resultdict[order.index(0)][2])
                images=[]#np.array(resultdict[order.index(0)][3])
                for i in range(threads):
                    images=images+resultdict[order.index(i)][3]
                    times=times+resultdict[order.index(i)][2]
726
                del resultdict
727
                return success,np.array(times),images
728
729
#        except Exception as E:
#            raise Exception(E)
730
            
731
    
732
    def download_raw_images_png_by_times_thread(port,times,out_q,threadnumber,version=0):
733
734
735
736
        images=[]
        time=[]
        successes=[]
        for i in times:
737
            imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
738
            if imag[0]:
739
                images.append(np.array(imag[1],dtype=np.uint16))
740
#                print(np.min(imag[1]),np.max(imag[1]))
741
742
743
744
745
746
                time.append(i)
                successes.append(True)
            else:
                successes.append(False)
        out_q.put([threadnumber,successes,time,images])        
    #    return success,np.array(time),np.array(images,dtype=np.uint16)
747
748
749
750
751
752
753
754
755
756
757
758
759
760
        
    def convert_raw_to_temp_thread(port,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
        images=[]
        time=[]
        successes=[]
        for i in times:
            imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
            if imag[0]:
                images.append(np.array(imag[1],dtype=np.uint16))
                time.append(i)
                successes.append(True)
            else:
                successes.append(False)
        if give_ERROR:
761
            sucess,images,error_images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
762
        else:
763
            success,images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
764
765
766
767
        
        for i in range(len(images)):
            images[i]=(images[i]*FOV).astype(np.float32)
        print(datetime.datetime.now(),"correcting bad pixels")
768
        images=IR_tools.correct_images(images,badpixels)
769
        out_q.put([threadnumber,successes,time,images]) 
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
    
    def NUC_raw_thread(port,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
        images=[]
        time=[]
        successes=[]
        for i in times:
            imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
            if imag[0]:
                images.append(np.array(imag[1],dtype=np.uint16))
                time.append(i)
                successes.append(True)
            else:
                successes.append(False)
        if give_ERROR:
            images,error_images=apply_NUC(images,gain,offset,gain_error,offset_error,give_ERROR)
        else:
            images=apply_NUC(images,gain,offset,gain_error,offset_error,give_ERROR)
        
        for i in range(len(images)):
            images[i]=(images[i]*FOV).astype(np.float32)
        print(datetime.datetime.now(),"correcting bad pixels")
        images=IR_tools.correct_images(images,badpixels)
        out_q.put([threadnumber,successes,time,images]) 
793
        
794
def download_last_raw_image_by_time(port,starttime,stoptime,version=0):
795
796
    OP=get_OP_by_time(starttime)
    Cam=portcamdict[OP]['AEF'+str(port)]
797
    if Cam.split("_")[0]=="Infratec" and OP=="OP1.2a":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
798
799
        if version==0:
            version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
800
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
801
    elif Cam.split("_")[0]=="IRCam"or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
802
803
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
804
805
806
807
808
809
810
811
812
813
814
815
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
    else:
        raise Exception("camera unknown, stopping here")
    try:
        res = urllib.request.urlopen(larchivepath+"/_signal.png?from="+str(starttime-10)+"&upto="+str(stoptime))
        img = Image.open(BytesIO(res.read()))
        res.close()
#        pixelarray = np.array(img.getdata()).reshape(img.size[1], img.size[0])
        pixelarray = np.array(img,dtype=np.uint16)#.swapaxes(0,1)
        return True, pixelarray
    except urllib.error.URLError as e:
        print(e)
816
        return False, [-1]
Holger Niemann's avatar
Holger Niemann committed
817
    
818
def download_raw_parlog_by_program(port,program,version=0):
819
820
821
822
823
824
825
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
826
        return download_raw_parlog_by_times(port,starttime,stoptime,version)    
827
    except:
Holger Niemann's avatar
Holger Niemann committed
828
829
830
        print("cannot find the program")
        return False,0,0
    
831
def download_raw_parlog_by_times(port,starttime,stoptime,version=0):
832
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
833
834
        if version==0:
            version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG")
Holger Niemann's avatar
Holger Niemann committed
835
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)
836
    elif port in [10,11,20,21,30,31,40,41,51]:#elif camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
837
838
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG")
Holger Niemann's avatar
Holger Niemann committed
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)   
    else:
        print("camera unknown, stopping here")
        raise Exception
    try:
        res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
        return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']
    

855
def get_INFRATEC_filter_by_program(program,version=0):
856
    prog=get_program_from_PID(program)
857
858
859
860
861
862
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
Holger Niemann's avatar
Holger Niemann committed
863
        return  get_INFRATEC_filter_by_times(starttime,stoptime,50,version)
864
    else:#except:
Holger Niemann's avatar
Holger Niemann committed
865
866
867
        print("cannot find the program")
        return False,0,0

868
def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0):
869
870
871
872
873
874
875
876
877
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
    if OP=="OP1.2a":
        if version==0:
            version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG")
        larchivepath=archivepath+"QRT_INFRATEC/AEF"+str(port)+"_raw_PARLOG/V"+str(version)
    elif OP=="OP1.2b":
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG")
        larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_raw_PARLOG/V"+str(version)
Holger Niemann's avatar
Holger Niemann committed
878
879
880
881
882
883
884
885
886
    try:
        res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
Holger Niemann's avatar
Holger Niemann committed
887
888
889
        return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']['filter']  
    else:
        return False, 0,-1
Holger Niemann's avatar
Holger Niemann committed
890
    
891
def get_exposure_by_program(port,program,version=0):
892
    prog=get_program_from_PID(program)
893
894
895
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
Holger Niemann's avatar
Holger Niemann committed
896
897
898
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
899
        return get_exposure_by_times(port,starttime,stoptime,version)
Holger Niemann's avatar
Holger Niemann committed
900
    else:
901
902
#    except Exception as E:
        print("cannot find the program, no exposure time available;")#, E)
Holger Niemann's avatar
Holger Niemann committed
903
904
        return False,0,0
        
905
def get_exposure_by_times(port,starttime,stoptime,version=0):
906
907
908
    OP=IR_tools.get_OP_by_time(time_ns=stoptime)
    if OP=="OP1.2a" and port==50:
#    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
909
910
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"   
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_" 
911
912
#    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
    else:
913
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_" 
914
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
915
    if version==0:
916
        version=get_latest_version(stream+"DATASTREAM")
917
    return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/3/exposuretime"+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
Holger Niemann's avatar
Holger Niemann committed
918
      
919
def get_camera_temp_by_program(port,program,version=0):
920
921
922
923
924
925
926
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
927
        return get_camera_temp_by_times(port,starttime,stoptime,version)
928
    except:
Holger Niemann's avatar
Holger Niemann committed
929
930
931
        print("cannot find the program")
        return False,0,0
        
932
def get_camera_temp_by_times(port,starttime,stoptime,version=0):
933
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
934
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"  
Holger Niemann's avatar
Holger Niemann committed
935
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
936
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
937
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
Holger Niemann's avatar
Holger Niemann committed
938
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
939
940
941
    else:
        print("Port unknown")
        raise Exception
942
943
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
944
    dummy=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
945
946
947
    if dummy[0]:
        return dummy
    else:
948
        return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratempetarure/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
Holger Niemann's avatar
Holger Niemann committed
949
    
950
def get_sensor_temp_by_program(port,program,version=0):
951
    prog=get_program_from_PID(program)
952
953
954
955
956
957
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
958
        return get_sensor_temp_by_times(port,starttime,stoptime,version)
959
    else:#except:
Holger Niemann's avatar
Holger Niemann committed
960
961
962
        print("cannot find the program")
        return False,0,0
        
963
def get_sensor_temp_by_times(port,starttime,stoptime,version=0):
964
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
965
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
Holger Niemann's avatar
Holger Niemann committed
966
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
967
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
968
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
Holger Niemann's avatar
Holger Niemann committed
969
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
970
971
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
972
    return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
Holger Niemann's avatar
Holger Niemann committed
973

974
def get_camera_framerate_by_program(port,program,version=0):
975
    prog=get_program_from_PID(program)
976
977
978
979
980
981
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
982
        return get_camera_framerate_by_times(port,starttime,stoptime,version)
983
984
#    except:
    else:
Holger Niemann's avatar
Holger Niemann committed
985
986
987
        print("cannot find the program")
        return False,0,0
        
988
def get_camera_framerate_by_times(port,starttime,stoptime,version=0):
989
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
990
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
Holger Niemann's avatar
Holger Niemann committed
991
        larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
992
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
993
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
Holger Niemann's avatar
Holger Niemann committed
994
        larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
995
996
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")    
997
    return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
Holger Niemann's avatar
Holger Niemann committed
998
999
    

1000
def get_frametype_by_program(port,program,version=0):
For faster browsing, not all history is shown. View entire blame