downloadversionIRdata.py 97.4 KB
Newer Older
Holger Niemann's avatar
Holger Niemann committed
1
2
3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
4
updated on Tue Aug 21 10:20:00 2018
Holger Niemann's avatar
Holger Niemann committed
5

6
Version: 2.7.0
7
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2) . #of updates to add functionalities . #number of updates for bug fixes )
Holger Niemann's avatar
Holger Niemann committed
8
9
10
11
@author: holn
"""

import numpy as np
12
import IR_image_tools as IR_tools
13
from IR_config_constants import archivepath,portcamdict,camlimdict,valid_FOV_circle,valid_background_rectangle,TC_port,TC_channel,IRCamColdframes_fittingpath,exJet,exJet_trans
14
15
16
17
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
Holger Niemann's avatar
Holger Niemann committed
18
19
20
import datetime
import urllib
import json
21
22
from PIL import Image
from io import BytesIO
23
from os.path import join
24
25
26
import matplotlib.pyplot as plt
try:        
    import multiprocessing    
27
    import archivedb as AKF_2
28
29
30
31
    fastDL=True
except Exception as E:
    print(E)
    fastDL=False
Holger Niemann's avatar
Holger Niemann committed
32

33

34
try:
35
    config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
36
37
38
39
40
    FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0]   
    del FOV
except:
    config_path=""

41
42
43
44
45
46
47
48
49
50
51
52
53
54
def get_OP_by_program(program):
    try:    
        bla=int(program.split(".")[0])
        time_ns=int(TimeToNs([bla // 10000,bla % 10000 // 100,bla % 10000 % 100],[10,0,0,0]))
        return get_OP_by_time(time_ns)
    except Exception as E:
        print(E)
        return None
    
def get_OP_by_time(time_ns):
    dateOP=datetime.datetime.utcfromtimestamp(time_ns/1e9)
    if dateOP.year==2017:
        if dateOP.month>8 and dateOP.month<12:
            return "OP1.2a"
55
        elif dateOP.month==8 and dateOP.day>=28:
56
57
58
59
60
61
62
63
64
65
66
67
68
            return "OP1.2a"
        elif dateOP.month==12 and dateOP.day<8:
            return "OP1.2a"
        else:
            return None        
    elif dateOP.year==2018:
        return "OP1.2b"
    elif dateOP.year<=2016 and dateOP.year>=2015:
        if (dateOP.year==2016 and dateOP.month<=3) or (dateOP.year==2015 and dateOP.month==12):
            return "OP1.1"
        else:
            return None

69
def get_latest_version(stream,project="W7X"):
70
71
72
73
74
     """Find out latest version of given stream
     :param stream   url stream of interest
     :return int of version number of None if non-versioned or non-existing stream
     author: G. Schlisio, edit by holn
     """
75
     request =urllib.request.Request(archivepath +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
     try:
         response = urllib.request.urlopen(request)
         d = json.loads(response.read().decode('utf-8'))
     except urllib.error.HTTPError as ex:
         msg = ex.read()
         raise RuntimeError(msg)
     # detect unversioned or non-existing stream
     if d["versionInfo"] == []:
         return None
     versions = []
     for i in d['versionInfo']:
         versions.append(i['number'])

     return max(versions)

Holger Niemann's avatar
Holger Niemann committed
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
def TimeToNs(date,time):
    """
    TimeToNs(date,time)
    (list)  date    [year,month,day]
    (list)  time    [hours,minutes,seconds,microseconds]
    """ 
    date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3])
    div=date_time-datetime.datetime(1970,1,1,0,0,0)
    nsdate=div.total_seconds()*1e9
    return nsdate

def read_program(timestampstart,timestamp_end=0,tol=60):
    """
    read_program()
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
    if timestamp_end!=0:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestamp_end+tol*1e9))
    else:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestampstart+tol*1e9))
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw=res.read()
        res.close()
    except urllib.error.URLError as e:
        print('!get_program: Error opening URL')
        print(e)
        return False,0
    else:
        prog_string=prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl=prog_list['programs'][0]
        return True, pl

def read_restdb_old(request_url):
    """
    Reads JSON data from W7-X REST API
    Returns: 
        valid: access ok
        t: numpy-array of time
        signal: numpy-array of requested signals
132
    by H. Thomsen
Holger Niemann's avatar
Holger Niemann committed
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
    """
    try:
        res = urllib.request.urlopen(request_url)
    except urllib.error.URLError as e:
        print(e)
        return False, 0, -1
    else:
        signal_raw=res.read()
        res.close()
        signal_string=signal_raw.decode(encoding='UTF-8')
        signal_list = json.loads(signal_string)
        signal0=np.array(signal_list['values'])
        t=np.array(signal_list['dimensions'])
        return True, t, signal0

148
def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
Holger Niemann's avatar
Holger Niemann committed
149
150
151
    """
    download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
        time in ns
152
    Have to swap 11, 21 until correction in the database
Holger Niemann's avatar
Holger Niemann committed
153
    """    
154
    project="W7X"
155
    OP=IR_tools.get_OP_by_time(time_ns=time)
156
157
158
159
    if port == 21:
        port = 11
    elif port == 11:
        port = 21
160
    if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
161
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
162
        larchivepath=archivepath+project+"/"+stream#"QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
Holger Niemann's avatar
Holger Niemann committed
163
        query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
164
    elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
165
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_LUT_"
166
        larchivepath=archivepath+project+"/"+stream
Holger Niemann's avatar
Holger Niemann committed
167
168
169
170
        query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
        print("camera unknown, stopping here")
        raise Exception
171
172
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
173
174
    if verbose>0:
        print("LUT V"+str(version)+" is used")
Holger Niemann's avatar
Holger Niemann committed
175
    #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
176
    LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
177
178
    if LUTpar[0]:
        LUTid=LUTpar[2][0]['structure'][query]
179
        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
180
        if LUTs[0]:
181
            LUTs=LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
Holger Niemann's avatar
Holger Niemann committed
182
183
184
185
            LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True,LUT
        else:
186
            print("Warning: unable to download the LUTs")
Holger Niemann's avatar
Holger Niemann committed
187
188
189
190
            del LUTpar, LUTs
            return False,0
    else:
        del LUTpar
191
        print("Warning: unable to find LUTs, check your request")
Holger Niemann's avatar
Holger Niemann committed
192
193
        return False,0

194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216

def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
    
    Read LUT V3 from local files for testing new calibration.
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
    OP = get_OP_by_time(this_time)
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
        with open(LUT_dir+"\\"+filename) as data_file:  
            jfile=json.load(data_file)
        LUT=np.array(jfile['LUT']).swapaxes(0,1)
        if verbose>0:
            print("succesfully loaded V3 LUT from local directory")
        return True, LUT
    except Exception as E:
        print("ERROR in loading V3 LUTs",E)
        return False, []
    
217
def download_NUC_by_program(port,program,exposure,version=0):
218
    prog=get_program_from_PID(program)
219
220
221
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
Holger Niemann's avatar
Holger Niemann committed
222
223
224
225
226
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_NUC_by_times(port,starttime,stoptime,exposure,version)
    else:
227
#    except:
Holger Niemann's avatar
Holger Niemann committed
228
229
230
        print("cannot find the program")
        return False,0,0

231
def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
232
233
    project="W7X"
    larchivepath=archivepath+project+"/"+"QRT_IRCAM/AEF"+str(port)+"_NUC_"
Holger Niemann's avatar
Holger Niemann committed
234
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
235
236
    if version==0:
        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_NUC_DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        
        n=0
        nuctimes=[0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp']==exposure:
                nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i=NUCpar['structure']['offset_index']                
            n+=1 
        if nuctimes[0]!=0:
255
            NUC=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
256
257
258
259
260
261
262
263
264
265
266
            if NUC[0]:
                images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)                    
                return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
            else:
                print("NUC image for requested exposure time not found")
                return False, 0, 0
        else:
            print("NUC image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
267

268
def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
269
    prog=get_program_from_PID(program)
270
271
272
273
274
275
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
276
        return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose-1)
277
278
279
280
281
    else:
#    except:
        print("cannot find the program")
        return False,0,0
    
282
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
283
284
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if OP=="OP1.2a":
285
286
287
288
        t1=endtime
        t0=starttime
        prog=read_program(t1)
        program=prog[1]['id']
289
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):
290
            
291
292
293
294
295
296
297
298
299
300
301
302
303
304
            print("rebuilding coldframe")
            #use any way the rebuilt coldframe.
            sT = get_sensor_temp_by_program(port, program)[2][0]
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("unable to download reference frames")
            filestring = 'AEF' + str(port) + '_et' + str(t_exp)
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
305
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
            gain_error=0
            offset_error=0        
            return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        elif (port == 11) or (port == 21):
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("unable to download reference frames")
            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
            if NUC_DL[0]==False:
                raise Exception("NUC was not found")
            else:
                ci = NUC_DL[1][2]
325
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1)
326
327
328
329
330
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)                                
            gain_error=0
            offset_error=0
331
            return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
        else:
            return download_NUC_by_times(port,starttime,endtime,t_exp)
    elif OP=="OP1.2b":
        gain_error=0
        offset_error=0
        gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
        if gotit:
            coldref=hotcold[1]
            hotref=hotcold[0]
        else:
            raise Exception("unable to download reference frames")
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
        exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
        if exist:
            cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
            for ele in coldframes:
                cold+=ele                
            cold=np.asarray(cold/len(coldframes),dtype=np.uint16)        
        else:
            return False,[0],[0]
352
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
353
354
355
356
357
358
359
360
361
362
        badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
        return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
    else:
        raise Exception("unknown Operation phase or NUC method not implemented for this OP")

def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0):
    """
    download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
    """
Holger Niemann's avatar
Holger Niemann committed
363
    gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(endtime-20e6))
364
365
366
367
368
369
370
371
    if gotit:
        expinds=np.where(texp_t==t_exp)[0]
        if len(expinds)==0:
            print("cannot find the exposure time in the given data")
            return False,[0],[0]                               
    else:
        print("exposure time not found")
        return False,[0],[0]
Holger Niemann's avatar
Holger Niemann committed
372
    gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(endtime-20e6))
373
    if gotitf:
Holger Niemann's avatar
Holger Niemann committed
374
        typiinds=np.where(values_f[expinds]==frametype)[0]
375
        if len(typiinds)>0:
Holger Niemann's avatar
Holger Niemann committed
376
377
378
            ref_t=[np.min(timef[expinds][typiinds]),np.max(timef[expinds][typiinds])]
#            print((ref_t[1]-ref_t[0])/1e9)
#            print(len(timef),len(timef[expinds]),len(timef[expinds][typiinds]))
379
380
381
382
383
384
385
        else:#okay the early data stuff or strange stuff
            print("frame type was not identified assuming that the first part is the cold and the second one the background")
            frametimes=time_t[expinds]
            diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
            turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
            if frametype==0:
                ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
Holger Niemann's avatar
Holger Niemann committed
386
                print((ref_t[1]-ref_t[0])/1e9)
387
388
389
            elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
                ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
Holger Niemann's avatar
Holger Niemann committed
390
                print((ref_t[1]-ref_t[0])/1e9)
391
392
393
394
395
396
397
            else:
                raise Exception("requested Frametype unknown and not implemented!")
    else:
        print("frametype not found")
        return False,[0],[0]
    t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
    t1date=t1date.isoformat()
Holger Niemann's avatar
Holger Niemann committed
398
    t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
399
400
401
    t0date=t0date.isoformat()
    if version==0:
        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_raw_DATASTREAM")
402
    larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
403
    timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
Holger Niemann's avatar
Holger Niemann committed
404
    t_offset=timest0[-1][0]-time_t[0]
405
    if t_offset>10e6:
Holger Niemann's avatar
Holger Niemann committed
406
        print("time offset detected, try to correct this, offset is",t_offset)
407
408
409
410
411
412
    else:
        t_offset=0
#            print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
    return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset)
        
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
413
    larchivepath=archivepath+"W7X/QRT_IRCAM/AEF"+str(port)+"_raw_"
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_raw_DATASTREAM")
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        

        COLDtime=0
        HOTtime=0
        try:
            COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
        except:
            print("cold image for requested exposure time not found")
        try:
            HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
        except:
            print("hot image for requested exposure time not found")
        images=[]
        if HOTtime!=0:
439
            HOT=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
440
441
442
            if HOT[0]:
                images.append(HOT[2])
        if COLDtime!=0:
443
            COLD=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
444
445
446
447
448
449
450
451
452
453
            if COLD[0]:
                images.append(COLD[2])
                
        if HOT[0] and COLD[0] and len(images)==2:
            return True,images,['hot','cold']
        else:
            print("hot and cold image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
Holger Niemann's avatar
Holger Niemann committed
454
    
455
def download_background_by_program(port,program,exposure,version=0):
456
457
458
    '''
    returned exist,time,frame
    '''
459
    prog=get_program_from_PID(program)
460
461
462
463
464
465
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
466
        return download_background_by_times(port,starttime,stoptime,exposure,version)
467
    else:#except:
Holger Niemann's avatar
Holger Niemann committed
468
469
470
        print("cannot find the program")
        return False,0,0
    
471
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
472
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
473
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_background_" 
474
        larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_background_"      
475
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
476
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_background_"   
477
        larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_background_"   
Holger Niemann's avatar
Holger Niemann committed
478
479
480
    else:
        print("camera unknown, stopping here")
        raise Exception
481
482
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
483
484
485
486
487
488
489
490
491
492
493
494
495
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
        n=0
        backtimes=[0]
        for backpar in signal_list['values']:
            if backpar['structure']['Texp']==exposure:
496
                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
497
498
499
500
501
502
                    if backpar['structure']['filter']==camera_filter:
                        backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
                else:
                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
            n+=1
        if backtimes[0]!=0:
503
            backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
504
505
506
507
508
509
510
511
512
513
514
            if backdat[0]:                                    
                return backdat#[True,backdat[2]]
            else:
                print("background image for requested exposure time(, filter) not found")
                return False,0,0
        else:
            print("background image for requested exposure time(, filter) not found")
            return False,0,0
    else:
        return False,0,0

515
def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False):
516
517
518
519
520
521
522
523
524
525
526
527
    "OP1.2b function"
    exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version)
    camera=portcamdict["OP1.2b"]["AEF"+str(port)]    
    background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
    for frame in backgroundframes:
        background+=frame
    background=np.asarray(background/len(backgroundframes),dtype=np.uint16)
    if plot_it:
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image unnuced")
    if not camera.split("_")[0]=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
528
        background=apply_NUC([background],gain,offset)[0]
529
    if plot_it:        
Holger Niemann's avatar
Holger Niemann committed
530
        
531
532
533
534
535
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image nuced")
    return exist,btime[0],background

536
def download_raw_images_by_program(port,program,time_s=0,version=0,verbose=0):
537
538
539
540
541
542
543
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
544
545
546
        if type(time_s)==list or type(time_s)==np.ndarray:
            if len(time_s)>2 or time_s[0]>1000:
                raise Exception("invalid input for the timewindow!")
Holger Niemann's avatar
Holger Niemann committed
547
548
549
550
            tstart=time_s[0]
            tstop=time_s[1]
            if tstop<tstart:
                raise Exception("endtime before starttime")
551
            return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
552
        else:
Holger Niemann's avatar
Holger Niemann committed
553
            if time_s==0:
554
                return download_raw_images_by_times(port,starttime,stoptime,version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
555
            else:
556
                return download_raw_images_by_times(port,starttime,int(starttime+time_s*1e9),version,verbose=verbose-1)
557
    except:
558
559
        if verbose>0:
            print("cannot find the program")
Holger Niemann's avatar
Holger Niemann committed
560
561
        return False,0,0

562
def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
563
564
565
566
567
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
    if OP=="OP1.2a":
        if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":        
            if version==0:
                version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
568
            larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
569
570
571
        elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
            if version==0:
                version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
572
            larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
573
        else:
574
575
            if verbose>0:
                print("camera unknown, stopping here")
576
577
            raise Exception
    elif OP=="OP1.2b":
578
579
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
580
        larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
581
    
582
583
    if (stoptime-starttime)/intervalSize>1:        
        nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
584
585
        if verbose>0:
            print("timewindow to large, splitting into smaller fractions ("+str(nrinterv)+")",(stoptime-starttime)/1e9)
Holger Niemann's avatar
Holger Niemann committed
586
        intervalls=[]
587
        for i in range(nrinterv):
588
            intervalls.append(int(starttime-10+i*intervalSize))
Holger Niemann's avatar
Holger Niemann committed
589
        intervalls.append(stoptime)
590
        for i in range(nrinterv):
Holger Niemann's avatar
Holger Niemann committed
591
            try:
Holger Niemann's avatar
Holger Niemann committed
592
                res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1]))
Holger Niemann's avatar
Holger Niemann committed
593
594
                signal_list = json.loads(res.read().decode('utf-8'))
                res.close()
595
596
597
                
                images=[np.array(ele, dtype=np.uint16) for ele in signal_list['values']]
#                signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
598
599
600
                t=np.array(signal_list['dimensions'])
                del signal_list
                if i==0:
601
                    allimages=images.copy()
Holger Niemann's avatar
Holger Niemann committed
602
603
                    time=t
                else:
604
                    allimages=allimages+images#np.append(ressignal,signal,axis=0)
Holger Niemann's avatar
Holger Niemann committed
605
                    time=np.append(time,t)
606
                del images,t
Holger Niemann's avatar
Holger Niemann committed
607
            except urllib.error.URLError as e:
608
609
                if verbose>0:
                    print(e)                
610
        return True,time,allimages
Holger Niemann's avatar
Holger Niemann committed
611
612
613
614
615
    else:
        try:
            res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
616
617
            images=[np.array(ele, dtype=np.uint16) for ele in signal_list['values']]
#            signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
618
619
            t=np.array(signal_list['dimensions'])
            del signal_list
620
            return True, t, images
Holger Niemann's avatar
Holger Niemann committed
621
        except urllib.error.URLError as e:
622
623
            if verbose>0:
                print(e)
Holger Niemann's avatar
Holger Niemann committed
624
            return False, 0,-1
625

626
if fastDL:
627
    def download_raw_images_by_program_via_png(port,program,time_s=0,version=0,threads=1, verbose=0):
628
        prog=get_program_from_PID(program)
629
630
631
#        try:
#            t_program = AKF_2.get_program_from_to(program)
#            prog =AKF_2.get_program_list(t_program[0], t_program[1])
632
633
634
635
636
        if not prog[0]:
            if verbose>0:
                print("program not found, cannot download the data")
                return False,[0],[0]
        else:
637
638
            starttime=prog[1]['trigger']['1'][0]
            stoptime=prog[1]['trigger']['6'][0]
639
640
641
            success=True
            OP=get_OP_by_time(starttime)
            Cam=portcamdict[OP]['AEF'+str(port)]
642
643
644
645
646
647
648
649
650
651
652
653
            if OP=="OP1.2a":
                if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
                    if version==0:
                        version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
                    larchivepath="Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
                elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
                    if version==0:
                        version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
                    larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
                else:
                    raise Exception("Port number does not fit the known cameras")
            elif OP=="OP1.2b":
654
655
                if version==0:
                    version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
656
                larchivepath="Test/raw/W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
Holger Niemann's avatar
Holger Niemann committed
657
658
659
660
661
662
663
664
            if type(time_s)==list:
                tstart=time_s[0]
                tstop=time_s[1]
                if tstop<tstart:
                    raise Exception("endtime before starttime")
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
                stdate=stdate.isoformat()
                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
665
666
                enddate=enddate.isoformat()
            else:
Holger Niemann's avatar
Holger Niemann committed
667
668
669
670
671
672
673
674
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
                stdate=stdate.isoformat()
                if time_s==0:
                    enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
                    enddate=enddate.isoformat()
                else:
                    enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)  
                    enddate=enddate.isoformat()
675
                #"2017-11-15 08:00:00"
676
677
678
679
680
            try:
                times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
            except Exception as E:
                raise Warning(E)
                return False,[0],[0]            
681
682
683
684
685
686
687
688
689
            time=[]
            images=[]
            lnt=len(times)
            if threads==1:
                for i in range(lnt):
                    ele=times[lnt-1-i]
                    imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
                    if imag[0]:
                        time.append(ele[0])
690
                        images.append(np.array(imag[1],dtype=np.uint16))
691
692
                    else:
                        success=False
693
                return success,np.array(time),images
694
695
696
697
698
699
700
701
702
703
704
705
            else:
                tim=[]
                for i in range(lnt):
                    tim.append(times[lnt-1-i][0])
                intervalls=[]
                intervalSize=int(lnt/threads)
                for i in range(threads):
                    intervalls.append(int(i*intervalSize))
                intervalls.append(lnt)
                jobs = []
                out_q=multiprocessing.Queue()
                for i in range(threads):
706
707
                    if verbose>0:
                        print("Start Thread ",i+1)                
708
709
710
711
712
713
714
715
                    p = multiprocessing.Process(target=download_raw_images_png_by_times_thread, args=(port,tim[intervalls[i]:intervalls[i+1]],out_q,i,version,))
                    jobs.append(p)
                    p.start()
                resultdict = []
                for i in range(threads):
                    resultdict.append(out_q.get())
                for p in jobs:
                    p.join()
716
717
                if verbose>0:
                    print("all threads are done")
718
719
720
721
722
723
                order=[]
                for ele in resultdict:
                    order.append(ele[0])
                    if len(np.where(np.asarray(ele[1])==False)[0])>0:
                        success=False
                
724
725
726
727
728
                times=[]#np.array(resultdict[order.index(0)][2])
                images=[]#np.array(resultdict[order.index(0)][3])
                for i in range(threads):
                    images=images+resultdict[order.index(i)][3]
                    times=times+resultdict[order.index(i)][2]
729
                del resultdict
730
                return success,np.array(times),images
731
732
#        except Exception as E:
#            raise Exception(E)
733
            
734
    
735
    def download_raw_images_png_by_times_thread(port,times,out_q,threadnumber,version=0):
736
737
738
739
        images=[]
        time=[]
        successes=[]
        for i in times:
740
            imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
741
            if imag[0]:
742
                images.append(np.array(imag[1],dtype=np.uint16))
743
#                print(np.min(imag[1]),np.max(imag[1]))
744
745
746
747
748
749
                time.append(i)
                successes.append(True)
            else:
                successes.append(False)
        out_q.put([threadnumber,successes,time,images])        
    #    return success,np.array(time),np.array(images,dtype=np.uint16)
750
751
752
753
754
755
756
757
758
759
760
761
762
763
        
    def convert_raw_to_temp_thread(port,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
        images=[]
        time=[]
        successes=[]
        for i in times:
            imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
            if imag[0]:
                images.append(np.array(imag[1],dtype=np.uint16))
                time.append(i)
                successes.append(True)
            else:
                successes.append(False)
        if give_ERROR:
764
            sucess,images,error_images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
765
        else:
766
            success,images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
767
768
769
770
        
        for i in range(len(images)):
            images[i]=(images[i]*FOV).astype(np.float32)
        print(datetime.datetime.now(),"correcting bad pixels")
771
        images=IR_tools.correct_images(images,badpixels)
772
        out_q.put([threadnumber,successes,time,images]) 
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
    
    def NUC_raw_thread(port,times,out_q,threadnumber,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
        images=[]
        time=[]
        successes=[]
        for i in times:
            imag=download_last_raw_image_by_time(port,i-10,i+10,version=version)
            if imag[0]:
                images.append(np.array(imag[1],dtype=np.uint16))
                time.append(i)
                successes.append(True)
            else:
                successes.append(False)
        if give_ERROR:
            images,error_images=apply_NUC(images,gain,offset,gain_error,offset_error,give_ERROR)
        else:
            images=apply_NUC(images,gain,offset,gain_error,offset_error,give_ERROR)
        
        for i in range(len(images)):
            images[i]=(images[i]*FOV).astype(np.float32)
        print(datetime.datetime.now(),"correcting bad pixels")
        images=IR_tools.correct_images(images,badpixels)
        out_q.put([threadnumber,successes,time,images]) 
796
        
797
def download_last_raw_image_by_time(port,starttime,stoptime,version=0):
798
799
    OP=get_OP_by_time(starttime)
    Cam=portcamdict[OP]['AEF'+str(port)]
800
    if Cam.split("_")[0]=="Infratec" and OP=="OP1.2a":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
801
802
        if version==0:
            version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
803
        larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
804
    elif Cam.split("_")[0]=="IRCam"or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
805
806
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
807
        larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
808
809
810
811
812
813
814
815
816
817
818
    else:
        raise Exception("camera unknown, stopping here")
    try:
        res = urllib.request.urlopen(larchivepath+"/_signal.png?from="+str(starttime-10)+"&upto="+str(stoptime))
        img = Image.open(BytesIO(res.read()))
        res.close()
#        pixelarray = np.array(img.getdata()).reshape(img.size[1], img.size[0])
        pixelarray = np.array(img,dtype=np.uint16)#.swapaxes(0,1)
        return True, pixelarray
    except urllib.error.URLError as e:
        print(e)
819
        return False, [-1]
Holger Niemann's avatar
Holger Niemann committed
820
    
821
def download_raw_parlog_by_program(port,program,version=0):
822
823
824
825
826
827
828
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
829
        return download_raw_parlog_by_times(port,starttime,stoptime,version)    
830
    except:
Holger Niemann's avatar
Holger Niemann committed
831
832
833
        print("cannot find the program")
        return False,0,0
    
834
def download_raw_parlog_by_times(port,starttime,stoptime,version=0):
835
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
836
837
        if version==0:
            version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG")
838
        larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)
839
    elif port in [10,11,20,21,30,31,40,41,51]:#elif camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
840
841
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG")
842
        larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)   
Holger Niemann's avatar
Holger Niemann committed
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
    else:
        print("camera unknown, stopping here")
        raise Exception
    try:
        res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
        return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']
    

858
def get_INFRATEC_filter_by_program(program,version=0):
859
    prog=get_program_from_PID(program)
860
861
862
863
864
865
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
Holger Niemann's avatar
Holger Niemann committed
866
        return  get_INFRATEC_filter_by_times(starttime,stoptime,50,version)
867
    else:#except:
Holger Niemann's avatar
Holger Niemann committed
868
869
870
        print("cannot find the program")
        return False,0,0

871
def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0):
872
873
874
875
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
    if OP=="OP1.2a":
        if version==0:
            version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG")
876
        larchivepath=archivepath+"W7X/QRT_INFRATEC/AEF"+str(port)+"_raw_PARLOG/V"+str(version)
877
878
879
    elif OP=="OP1.2b":
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG")
880
        larchivepath=archivepath+"W7X/QRT_IRCAM/AEF"+str(port)+"_raw_PARLOG/V"+str(version)
Holger Niemann's avatar
Holger Niemann committed
881
882
883
884
885
886
887
888
889
    try:
        res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
Holger Niemann's avatar
Holger Niemann committed
890
891
892
        return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']['filter']  
    else:
        return False, 0,-1
Holger Niemann's avatar
Holger Niemann committed
893
    
894
def get_exposure_by_program(port,program,version=0):
895
    prog=get_program_from_PID(program)
896
897
898
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
Holger Niemann's avatar
Holger Niemann committed
899
900
901
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
902
        return get_exposure_by_times(port,starttime,stoptime,version)
Holger Niemann's avatar
Holger Niemann committed
903
    else:
904
905
#    except Exception as E:
        print("cannot find the program, no exposure time available;")#, E)
Holger Niemann's avatar
Holger Niemann committed
906
907
        return False,0,0
        
908
def get_exposure_by_times(port,starttime,stoptime,version=0):
909
910
911
    OP=IR_tools.get_OP_by_time(time_ns=stoptime)
    if OP=="OP1.2a" and port==50:
#    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
912
        larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"   
913
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_" 
914
915
#    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
    else:
916
        larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_meta_" 
917
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
918
    if version==0:
919
        version=get_latest_version(stream+"DATASTREAM")
920
    return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/3/exposuretime"+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
Holger Niemann's avatar
Holger Niemann committed
921
      
922
def get_camera_temp_by_program(port,program,version=0):
923
924
925
926
927
928
929
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
930
        return get_camera_temp_by_times(port,starttime,stoptime,version)
931
    except:
Holger Niemann's avatar
Holger Niemann committed
932
933
934
        print("cannot find the program")
        return False,0,0
        
935
def get_camera_temp_by_times(port,starttime,stoptime,version=0):
936
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
937
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"  
938
        larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
939
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
940
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
941
        larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
942
943
944
    else:
        print("Port unknown")
        raise Exception
945
946
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
947
    dummy=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
948
949
950
    if dummy[0]:
        return dummy
    else:
951
        return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratempetarure/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
Holger Niemann's avatar
Holger Niemann committed
952
    
953
def get_sensor_temp_by_program(port,program,version=0):
954
    prog=get_program_from_PID(program)
955
956
957
958
959
960
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
961
        return get_sensor_temp_by_times(port,starttime,stoptime,version)
962
    else:#except:
Holger Niemann's avatar
Holger Niemann committed
963
964
965
        print("cannot find the program")
        return False,0,0
        
966
def get_sensor_temp_by_times(port,starttime,stoptime,version=0):
967
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
968
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
969
        larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
970
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
971
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
972
        larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
973
974
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
975
    return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
Holger Niemann's avatar
Holger Niemann committed
976

977
def get_camera_framerate_by_program(port,program,version=0):
978
    prog=get_program_from_PID(program)
979
980
981
982
983
984
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['1'][0]
        stoptime=prog[1]['trigger']['6'][0]
985
        return get_camera_framerate_by_times(port,starttime,stoptime,version)
986
987
#    except:
    else:
Holger Niemann's avatar
Holger Niemann committed
988
989
990
        print("cannot find the program")
        return False,0,0
        
991
def get_camera_framerate_by_times(port,starttime,stoptime,version=0):
992
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
993
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
994
        larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"      
995
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
996
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
997
        larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
998
999
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")    
1000
    return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
For faster browsing, not all history is shown. View entire blame