downloadversionIRdata.py 111 KB
Newer Older
Holger Niemann's avatar
Holger Niemann committed
1
2
3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
4
updated on Tue Aug 21 10:20:00 2018
Holger Niemann's avatar
Holger Niemann committed
5

6
Version: 2.9.1
7
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
Holger Niemann's avatar
Holger Niemann committed
8
9
10
11
@author: holn
"""

import numpy as np
12
import IR_image_tools as IR_tools
13
from IR_config_constants import archivepath,portcamdict,camlimdict,valid_FOV_circle,valid_background_rectangle,TC_port,TC_channel,IRCamColdframes_fittingpath,exJet,exJet_trans
14
15
16
17
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
Holger Niemann's avatar
Holger Niemann committed
18
19
20
import datetime
import urllib
import json
21
22
from PIL import Image
from io import BytesIO
23
from os.path import join
24
import matplotlib.pyplot as plt
25
26
try:          
    import threading
27
    import archivedb as AKF_2
28
29
30
31
    fastDL=True
except Exception as E:
    print(E)
    fastDL=False
Holger Niemann's avatar
Holger Niemann committed
32

33

34
try:
35
    config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
36
37
38
39
40
    FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0]   
    del FOV
except:
    config_path=""

41
42
43
44
45
46
47
48
49
50
51
52
53
54
def get_OP_by_program(program):
    try:    
        bla=int(program.split(".")[0])
        time_ns=int(TimeToNs([bla // 10000,bla % 10000 // 100,bla % 10000 % 100],[10,0,0,0]))
        return get_OP_by_time(time_ns)
    except Exception as E:
        print(E)
        return None
    
def get_OP_by_time(time_ns):
    dateOP=datetime.datetime.utcfromtimestamp(time_ns/1e9)
    if dateOP.year==2017:
        if dateOP.month>8 and dateOP.month<12:
            return "OP1.2a"
55
        elif dateOP.month==8 and dateOP.day>=28:
56
57
58
59
60
61
62
63
64
65
66
67
68
            return "OP1.2a"
        elif dateOP.month==12 and dateOP.day<8:
            return "OP1.2a"
        else:
            return None        
    elif dateOP.year==2018:
        return "OP1.2b"
    elif dateOP.year<=2016 and dateOP.year>=2015:
        if (dateOP.year==2016 and dateOP.month<=3) or (dateOP.year==2015 and dateOP.month==12):
            return "OP1.1"
        else:
            return None

69
def get_latest_version(stream,project="W7X"):
70
71
72
73
74
     """Find out latest version of given stream
     :param stream   url stream of interest
     :return int of version number of None if non-versioned or non-existing stream
     author: G. Schlisio, edit by holn
     """
75
     request =urllib.request.Request(archivepath +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
     try:
         response = urllib.request.urlopen(request)
         d = json.loads(response.read().decode('utf-8'))
     except urllib.error.HTTPError as ex:
         msg = ex.read()
         raise RuntimeError(msg)
     # detect unversioned or non-existing stream
     if d["versionInfo"] == []:
         return None
     versions = []
     for i in d['versionInfo']:
         versions.append(i['number'])

     return max(versions)

Holger Niemann's avatar
Holger Niemann committed
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
def TimeToNs(date,time):
    """
    TimeToNs(date,time)
    (list)  date    [year,month,day]
    (list)  time    [hours,minutes,seconds,microseconds]
    """ 
    date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3])
    div=date_time-datetime.datetime(1970,1,1,0,0,0)
    nsdate=div.total_seconds()*1e9
    return nsdate

def read_program(timestampstart,timestamp_end=0,tol=60):
    """
    read_program()
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
    if timestamp_end!=0:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestamp_end+tol*1e9))
    else:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestampstart+tol*1e9))
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw=res.read()
        res.close()
    except urllib.error.URLError as e:
        print('!get_program: Error opening URL')
        print(e)
        return False,0
    else:
        prog_string=prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl=prog_list['programs'][0]
        return True, pl

def read_restdb_old(request_url):
    """
    Reads JSON data from W7-X REST API
    Returns: 
        valid: access ok
        t: numpy-array of time
        signal: numpy-array of requested signals
132
    by H. Thomsen
Holger Niemann's avatar
Holger Niemann committed
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
    """
    try:
        res = urllib.request.urlopen(request_url)
    except urllib.error.URLError as e:
        print(e)
        return False, 0, -1
    else:
        signal_raw=res.read()
        res.close()
        signal_string=signal_raw.decode(encoding='UTF-8')
        signal_list = json.loads(signal_string)
        signal0=np.array(signal_list['values'])
        t=np.array(signal_list['dimensions'])
        return True, t, signal0

148
def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
Holger Niemann's avatar
Holger Niemann committed
149
150
151
    """
    download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
        time in ns
152
    Have to swap 11, 21 until correction in the database
Holger Niemann's avatar
Holger Niemann committed
153
    """    
154
    project="W7X"
155
    OP=IR_tools.get_OP_by_time(time_ns=time)
156
157
158
159
    if port == 21:
        port = 11
    elif port == 11:
        port = 21
160
    if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
161
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
162
        larchivepath=archivepath+project+"/"+stream#"QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
Holger Niemann's avatar
Holger Niemann committed
163
        query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
164
    elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
165
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_LUT_"
166
        larchivepath=archivepath+project+"/"+stream
Holger Niemann's avatar
Holger Niemann committed
167
168
169
170
        query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
        print("camera unknown, stopping here")
        raise Exception
171
172
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
173
174
    if verbose>0:
        print("LUT V"+str(version)+" is used")
Holger Niemann's avatar
Holger Niemann committed
175
    #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
176
    LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
177
178
    if LUTpar[0]:
        LUTid=LUTpar[2][0]['structure'][query]
179
        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
180
        if LUTs[0]:
181
            LUTs=LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
Holger Niemann's avatar
Holger Niemann committed
182
183
184
185
            LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True,LUT
        else:
186
            print("Warning: unable to download the LUTs")
Holger Niemann's avatar
Holger Niemann committed
187
188
189
190
            del LUTpar, LUTs
            return False,0
    else:
        del LUTpar
191
        print("Warning: unable to find LUTs, check your request")
Holger Niemann's avatar
Holger Niemann committed
192
193
        return False,0

194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216

def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
    
    Read LUT V3 from local files for testing new calibration.
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
    OP = get_OP_by_time(this_time)
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
        with open(LUT_dir+"\\"+filename) as data_file:  
            jfile=json.load(data_file)
        LUT=np.array(jfile['LUT']).swapaxes(0,1)
        if verbose>0:
            print("succesfully loaded V3 LUT from local directory")
        return True, LUT
    except Exception as E:
        print("ERROR in loading V3 LUTs",E)
        return False, []
    
217
def download_NUC_by_program(port,program,exposure,version=0):
218
    prog=get_program_from_PID(program)
219
220
221
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
Holger Niemann's avatar
Holger Niemann committed
222
223
224
225
226
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_NUC_by_times(port,starttime,stoptime,exposure,version)
    else:
227
#    except:
Holger Niemann's avatar
Holger Niemann committed
228
229
230
        print("cannot find the program")
        return False,0,0

231
def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
232
233
    project="W7X"
    larchivepath=archivepath+project+"/"+"QRT_IRCAM/AEF"+str(port)+"_NUC_"
Holger Niemann's avatar
Holger Niemann committed
234
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
235
236
    if version==0:
        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_NUC_DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        
        n=0
        nuctimes=[0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp']==exposure:
                nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i=NUCpar['structure']['offset_index']                
            n+=1 
        if nuctimes[0]!=0:
255
            NUC=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
256
257
258
259
260
261
262
263
264
265
266
            if NUC[0]:
                images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)                    
                return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
            else:
                print("NUC image for requested exposure time not found")
                return False, 0, 0
        else:
            print("NUC image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
267

268
def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
269
    prog=get_program_from_PID(program)
270
271
272
273
274
275
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
276
        return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose-1)
277
278
279
280
281
    else:
#    except:
        print("cannot find the program")
        return False,0,0
    
282
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
283
284
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if OP=="OP1.2a":
285
286
287
288
        t1=endtime
        t0=starttime
        prog=read_program(t1)
        program=prog[1]['id']
289
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):
290
            
291
292
293
294
295
296
297
298
299
300
301
302
303
304
            print("rebuilding coldframe")
            #use any way the rebuilt coldframe.
            sT = get_sensor_temp_by_program(port, program)[2][0]
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("unable to download reference frames")
            filestring = 'AEF' + str(port) + '_et' + str(t_exp)
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
305
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
            gain_error=0
            offset_error=0        
            return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        elif (port == 11) or (port == 21):
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("unable to download reference frames")
            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
            if NUC_DL[0]==False:
                raise Exception("NUC was not found")
            else:
                ci = NUC_DL[1][2]
325
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1)
326
327
328
329
330
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)                                
            gain_error=0
            offset_error=0
331
            return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
        else:
            return download_NUC_by_times(port,starttime,endtime,t_exp)
    elif OP=="OP1.2b":
        gain_error=0
        offset_error=0
        gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
        if gotit:
            coldref=hotcold[1]
            hotref=hotcold[0]
        else:
            raise Exception("unable to download reference frames")
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
        exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
        if exist:
            cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
            for ele in coldframes:
                cold+=ele                
            cold=np.asarray(cold/len(coldframes),dtype=np.uint16)        
        else:
            return False,[0],[0]
352
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
353
354
355
356
357
358
359
360
361
362
        badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
        return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
    else:
        raise Exception("unknown Operation phase or NUC method not implemented for this OP")

def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0):
    """
    download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
    """
Holger Niemann's avatar
Holger Niemann committed
363
    gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(endtime-20e6))
364
365
366
367
368
369
370
371
    if gotit:
        expinds=np.where(texp_t==t_exp)[0]
        if len(expinds)==0:
            print("cannot find the exposure time in the given data")
            return False,[0],[0]                               
    else:
        print("exposure time not found")
        return False,[0],[0]
Holger Niemann's avatar
Holger Niemann committed
372
    gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(endtime-20e6))
373
    if gotitf:
Holger Niemann's avatar
Holger Niemann committed
374
        typiinds=np.where(values_f[expinds]==frametype)[0]
375
        if len(typiinds)>0:
Holger Niemann's avatar
Holger Niemann committed
376
377
378
            ref_t=[np.min(timef[expinds][typiinds]),np.max(timef[expinds][typiinds])]
#            print((ref_t[1]-ref_t[0])/1e9)
#            print(len(timef),len(timef[expinds]),len(timef[expinds][typiinds]))
379
380
381
382
383
384
385
        else:#okay the early data stuff or strange stuff
            print("frame type was not identified assuming that the first part is the cold and the second one the background")
            frametimes=time_t[expinds]
            diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
            turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
            if frametype==0:
                ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
Holger Niemann's avatar
Holger Niemann committed
386
                print((ref_t[1]-ref_t[0])/1e9)
387
388
389
            elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
                ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
Holger Niemann's avatar
Holger Niemann committed
390
                print((ref_t[1]-ref_t[0])/1e9)
391
392
393
394
395
396
397
            else:
                raise Exception("requested Frametype unknown and not implemented!")
    else:
        print("frametype not found")
        return False,[0],[0]
    t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
    t1date=t1date.isoformat()
Holger Niemann's avatar
Holger Niemann committed
398
    t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
399
400
401
    t0date=t0date.isoformat()
    if version==0:
        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_raw_DATASTREAM")
402
    larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
403
    timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
Holger Niemann's avatar
Holger Niemann committed
404
    t_offset=timest0[-1][0]-time_t[0]
405
    if t_offset>10e6:
Holger Niemann's avatar
Holger Niemann committed
406
        print("time offset detected, try to correct this, offset is",t_offset)
407
408
409
410
411
412
    else:
        t_offset=0
#            print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
    return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset)
        
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
413
    larchivepath=archivepath+"W7X/QRT_IRCAM/AEF"+str(port)+"_raw_"
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
        version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_raw_DATASTREAM")
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        

        COLDtime=0
        HOTtime=0
        try:
            COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
        except:
            print("cold image for requested exposure time not found")
        try:
            HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
        except:
            print("hot image for requested exposure time not found")
        images=[]
        if HOTtime!=0:
439
            HOT=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
440
441
442
            if HOT[0]:
                images.append(HOT[2])
        if COLDtime!=0:
443
            COLD=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
444
445
446
447
448
449
450
451
452
453
            if COLD[0]:
                images.append(COLD[2])
                
        if HOT[0] and COLD[0] and len(images)==2:
            return True,images,['hot','cold']
        else:
            print("hot and cold image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
Holger Niemann's avatar
Holger Niemann committed
454
    
455
def download_background_by_program(port,program,exposure,version=0):
456
457
458
    '''
    returned exist,time,frame
    '''
459
    prog=get_program_from_PID(program)
460
461
462
463
464
465
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
466
        return download_background_by_times(port,starttime,stoptime,exposure,version)
467
    else:#except:
Holger Niemann's avatar
Holger Niemann committed
468
469
470
        print("cannot find the program")
        return False,0,0
    
471
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
472
    if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
473
        stream="QRT_INFRATEC/"+"AEF"+str(port)+"_background_" 
474
        larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_background_"      
475
    elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
476
        stream="QRT_IRCAM/"+"AEF"+str(port)+"_background_"   
477
        larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_background_"   
Holger Niemann's avatar
Holger Niemann committed
478
479
480
    else:
        print("camera unknown, stopping here")
        raise Exception
481
482
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
483
484
485
486
487
488
489
490
491
492
493
494
495
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
        n=0
        backtimes=[0]
        for backpar in signal_list['values']:
            if backpar['structure']['Texp']==exposure:
496
                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
497
498
499
500
501
502
                    if backpar['structure']['filter']==camera_filter:
                        backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
                else:
                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
            n+=1
        if backtimes[0]!=0:
503
            backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
504
505
506
507
508
509
510
511
512
513
514
            if backdat[0]:                                    
                return backdat#[True,backdat[2]]
            else:
                print("background image for requested exposure time(, filter) not found")
                return False,0,0
        else:
            print("background image for requested exposure time(, filter) not found")
            return False,0,0
    else:
        return False,0,0

515
def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False):
516
517
518
519
520
521
522
523
524
525
526
527
    "OP1.2b function"
    exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version)
    camera=portcamdict["OP1.2b"]["AEF"+str(port)]    
    background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
    for frame in backgroundframes:
        background+=frame
    background=np.asarray(background/len(backgroundframes),dtype=np.uint16)
    if plot_it:
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image unnuced")
    if not camera.split("_")[0]=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
528
        background=apply_NUC([background],gain,offset)[0]
529
    if plot_it:        
Holger Niemann's avatar
Holger Niemann committed
530
        
531
532
533
534
535
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image nuced")
    return exist,btime[0],background

536
def download_raw_images_by_program(port,program,time_s=0,version=0,verbose=0):
537
538
539
540
541
542
543
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
544
545
546
        if type(time_s)==list or type(time_s)==np.ndarray:
            if len(time_s)>2 or time_s[0]>1000:
                raise Exception("invalid input for the timewindow!")
Holger Niemann's avatar
Holger Niemann committed
547
548
549
550
            tstart=time_s[0]
            tstop=time_s[1]
            if tstop<tstart:
                raise Exception("endtime before starttime")
551
            return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
552
        else:
Holger Niemann's avatar
Holger Niemann committed
553
            if time_s==0:
554
                return download_raw_images_by_times(port,starttime,stoptime,version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
555
            else:
556
                return download_raw_images_by_times(port,starttime,int(starttime+time_s*1e9),version,verbose=verbose-1)
557
    except:
558
559
        if verbose>0:
            print("cannot find the program")
Holger Niemann's avatar
Holger Niemann committed
560
561
        return False,0,0

562
def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
563
564
565
566
567
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
    if OP=="OP1.2a":
        if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":        
            if version==0:
                version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
568
            larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
569
570
571
        elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
            if version==0:
                version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
572
            larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
573
        else:
574
575
            if verbose>0:
                print("camera unknown, stopping here")
576
577
            raise Exception
    elif OP=="OP1.2b":
578
579
        if version==0:
            version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
580
        larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
581
    
582
583
    if (stoptime-starttime)/intervalSize>1:        
        nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
584
585
        if verbose>0:
            print("timewindow to large, splitting into smaller fractions ("+str(nrinterv)+")",(stoptime-starttime)/1e9)
Holger Niemann's avatar
Holger Niemann committed
586
        intervalls=[]
587
        for i in range(nrinterv):
588
            intervalls.append(int(starttime-10+i*intervalSize))
Holger Niemann's avatar
Holger Niemann committed
589
        intervalls.append(stoptime)
590
        for i in range(nrinterv):
Holger Niemann's avatar
Holger Niemann committed
591
            try:
Holger Niemann's avatar
Holger Niemann committed
592
                res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1]))
Holger Niemann's avatar
Holger Niemann committed
593
594
                signal_list = json.loads(res.read().decode('utf-8'))
                res.close()
595
596
597
                
                images=[np.array(ele, dtype=np.uint16) for ele in signal_list['values']]
#                signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
598
599
600
                t=np.array(signal_list['dimensions'])
                del signal_list
                if i==0:
601
                    allimages=images.copy()
Holger Niemann's avatar
Holger Niemann committed
602
603
                    time=t
                else:
604
                    allimages=allimages+images#np.append(ressignal,signal,axis=0)
Holger Niemann's avatar
Holger Niemann committed
605
                    time=np.append(time,t)
606
                del images,t
Holger Niemann's avatar
Holger Niemann committed
607
            except urllib.error.URLError as e:
608
609
                if verbose>0:
                    print(e)                
610
        return True,time,allimages
Holger Niemann's avatar
Holger Niemann committed
611
612
613
614
615
    else:
        try:
            res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
616
617
            images=[np.array(ele, dtype=np.uint16) for ele in signal_list['values']]
#            signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
618
619
            t=np.array(signal_list['dimensions'])
            del signal_list
620
            return True, t, images
Holger Niemann's avatar
Holger Niemann committed
621
        except urllib.error.URLError as e:
622
623
            if verbose>0:
                print(e)
Holger Niemann's avatar
Holger Niemann committed
624
            return False, 0,-1
625

626
if fastDL:
627
    def download_raw_images_by_program_via_png(port,program,time_s=0,version=0,threads=1, verbose=0):
628
        prog=get_program_from_PID(program)
629
630
631
#        try:
#            t_program = AKF_2.get_program_from_to(program)
#            prog =AKF_2.get_program_list(t_program[0], t_program[1])
632
633
634
635
636
        if not prog[0]:
            if verbose>0:
                print("program not found, cannot download the data")
                return False,[0],[0]
        else:
637
638
            starttime=prog[1]['trigger']['1'][0]
            stoptime=prog[1]['trigger']['6'][0]
639
640
            OP=get_OP_by_time(starttime)
            Cam=portcamdict[OP]['AEF'+str(port)]
641
642
643
644
            if OP=="OP1.2a":
                if Cam.split("_")[0]=="Infratec":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
                    if version==0:
                        version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
645
                    larchivepath=archivepath+"Test/raw/W7X/"+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
646
647
648
                elif Cam.split("_")[0]=="IRCam":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
                    if version==0:
                        version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
649
                    larchivepath=archivepath+"W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
650
651
652
                else:
                    raise Exception("Port number does not fit the known cameras")
            elif OP=="OP1.2b":
653
654
                if version==0:
                    version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
655
656
657
658
659
                larchivepath=archivepath+"W7X/"+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
            return download_raw_images_by_time_via_png(larchivepath,starttime,stoptime,time_s=time_s,threads=threads,version=version,verbose=verbose-1)
            
    def download_raw_images_by_time_via_png(larchivepath,starttime,stoptime,time_s=0,threads=1,version=0,verbose=0):            
            success=True            
Holger Niemann's avatar
Holger Niemann committed
660
661
662
663
664
665
666
667
            if type(time_s)==list:
                tstart=time_s[0]
                tstop=time_s[1]
                if tstop<tstart:
                    raise Exception("endtime before starttime")
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
                stdate=stdate.isoformat()
                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
668
669
                enddate=enddate.isoformat()
            else:
Holger Niemann's avatar
Holger Niemann committed
670
671
672
673
674
675
676
677
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
                stdate=stdate.isoformat()
                if time_s==0:
                    enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
                    enddate=enddate.isoformat()
                else:
                    enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)  
                    enddate=enddate.isoformat()
678
                #"2017-11-15 08:00:00"
679
680
681
            try:
                times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
            except Exception as E:
682
                print(larchivepath)
683
684
                raise Warning(E)
                return False,[0],[0]            
685
686
687
688
689
690
            time=[]
            images=[]
            lnt=len(times)
            if threads==1:
                for i in range(lnt):
                    ele=times[lnt-1-i]
691
                    imag=download_last_raw_image_by_time(larchivepath,ele[0]-10,ele[0]+10)
692
693
                    if imag[0]:
                        time.append(ele[0])
694
                        images.append(np.array(imag[1],dtype=np.uint16))
695
696
                    else:
                        success=False
697
                return success,np.array(time),images
698
699
700
701
702
703
704
705
706
707
            else:
                tim=[]
                for i in range(lnt):
                    tim.append(times[lnt-1-i][0])
                intervalls=[]
                intervalSize=int(lnt/threads)
                for i in range(threads):
                    intervalls.append(int(i*intervalSize))
                intervalls.append(lnt)
                jobs = []
708
                resultdict = []
709
                for i in range(threads):
710
711
                    if verbose>0:
                        print("Start Thread ",i+1)                
712
713
714
                    TH=download_images_thread(i,larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,version)
                    jobs.append(TH)
                    TH.start()
715
716
                for p in jobs:
                    p.join()
717
718
                if verbose>0:
                    print("all threads are done")
719
720
721
722
                order=[]
                for ele in resultdict:
                    order.append(ele[0])
                    if len(np.where(np.asarray(ele[1])==False)[0])>0:
723
                        success=False                
724
725
726
727
728
                times=[]#np.array(resultdict[order.index(0)][2])
                images=[]#np.array(resultdict[order.index(0)][3])
                for i in range(threads):
                    images=images+resultdict[order.index(i)][3]
                    times=times+resultdict[order.index(i)][2]
729
                    resultdict[order.index(i)]=[]
730
                del resultdict
731
                return success,np.array(times),images
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
      
    class download_images_thread(threading.Thread):
        def __init__(self, threadID, larchivepath,times,resultlist,version=0):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            self.resultlist.append([self.threadID,successes,time,images])
754
    
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
    class convert_raw_to_temp_thread(threading.Thread):
        def __init__(self,larchivepath,times,resultlist,threadID,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
        def run(self):
            images=[]
            time=[]
            successes=[]        
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            if self.give_ERROR:
                sucess,images,error_images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR)
787
            else:
788
789
790
791
792
793
794
                success,images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR)
            
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
            print(datetime.datetime.now(),"correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels)
            self.resultlist.append([self.threadID,successes,time,images])
795
    
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
    class NUC_raw_thread(threading.Thread):
        def __init__ (self,larchivepath,times,resultlist,threadID,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            if self.give_ERROR:
                images,error_images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
828
            else:
829
830
831
832
833
834
835
                images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
            
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
            print(datetime.datetime.now(),"correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels)
            self.resultlist.append([self.threadID,successes,time,images]) 
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882

def download_raw_FLIR_images_via_png(t1,t6,time_s=0,version=0,threads=1,verbose=0):    
    larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
    if type(time_s)==list:
        tstart=time_s[0]
        tstop=time_s[1]
        if tstop<tstart:
            raise Exception("endtime before starttime")
        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
        stdate=stdate.isoformat()
        enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)  
        enddate=enddate.isoformat()
    else:
        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
        stdate=stdate.isoformat()
        if time_s==0:
            enddate=datetime.datetime.utcfromtimestamp(t6/1e9)        
            enddate=enddate.isoformat()
        else:
            enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_s)  
            enddate=enddate.isoformat()
        #"2017-11-15 08:00:00"
    typ1_version=get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM")
    if typ1_version==None:
        mode=1
    else:
        mode=2
    if mode==1:
        ### okay the data is stored probably as different sets (so recorded with FLIR software)
        timess,versions=get_FLIR_Pset_times(stdate,enddate,version)
        ### we not know the times for all presets for the program, now we have to download them, if times is not empty
        exist,time,frames,tsets=download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=verbose-1)            
    elif mode==2:##the data is stored only in one datastream, at least this datastream exist
        raise Exception("not implemented")
#        time=[]
#        images=[]
#        lnt=len(times)
#        if threads==1:
#            for i in range(lnt):
#                ele=times[lnt-1-i]
#                imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
#                if imag[0]:
#                    time.append(ele[0])
#                    images.append(np.array(imag[1],dtype=np.uint16))
#                else:
#                    success=False
#            return success,np.array(time),images
883
    else:
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
        raise Exception("cannot identifie the way the searched data is stored!")                        

    return exist,time,frames,mode,tsets,versions

def get_FLIR_Pset_times(stdate,enddate,version=0):
    larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
    if version==0:
        versions=[]
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS0_DATASTREAM"))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS1_DATASTREAM"))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS2_DATASTREAM"))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS3_DATASTREAM"))
    else:
        versions=[version,version,version,version]
    timess=[]
    if version==0:##no version specified search for data in all versions            
        counter=-1
        for version0 in versions:
            counter+=1
            if version0>0:
                found=False
                while (version0>0 and not found):                    
                    try:            
                        times_0=AKF_2.get_time_intervals(larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw",stdate.replace("T"," "),enddate.replace("T"," "))#
                        found=True
                    except Exception as E:
                        print(E)
                        found=False
                        times_0=[]
                        version0=version0-1
                timess.append(times_0)
            else:
                timess.append([])
    else:
        counter=-1
        for version0 in versions:
            counter+=1
            try:            
                times_0=AKF_2.get_time_intervals(larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw",stdate.replace("T"," "),enddate.replace("T"," "))#
            except:
                times_0=[]
            timess.append(times_0)
    return timess,versions

def download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=0):
    if len(timess[0])==len(timess[1]) and len(timess[2])==len(timess[3]) and len(timess[0])==0:
            exist=False
            time=[]
            frames=[]
            tsets=[]
    else:#timess is not empty, there is data in at least one Pset
        if verbose>0:
            print(datetime.datetime.now(),"data found, start the download")
        exist=True
        timesets=[]
        framess=[]
        success=True
        Psets=[]
        if threads==1:            
            counter=-1
            setn=-1
            for times in timess:                    
                setn+=1
                if times!=[]:
                    counter+=1
                    Psets.append(counter)
                    framess.append([])
                    timesets.append([])
                    lnt=len(times)                    
                    for i in range(lnt):                        
                        ele=times[lnt-1-i]
                        imag=download_last_raw_image_by_time(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw",ele[0]-10,ele[0]+10)
                        if imag[0]:
                            timesets[counter].append([ele[0],i,setn,counter])
                            framess[counter].append(np.array(imag[1],dtype=np.uint16))
                        else:
                            success=False
        else:#several threads, each for a preset
            tim=[]
            counter=-1
            setn=-1
            threads=0
            larchivepaths=[]
            versionss=[]
            for times in timess:                    
                setn+=1
                if times!=[]:
                    counter+=1
                    threads+=1
                    Psets.append(counter)
                    larchivepaths.append(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw")
                    tim.append(times)
                    versionss.append(versions[setn])
            jobs = []
978
            resultdict = []
979
980
981
            for i in range(threads):
                if verbose>0:
                    print("Start Thread ",i+1)                
982
983
984
                TH=download_images_thread(i,larchivepaths[i],tim[i],resultdict,versionss[i])
                jobs.append(TH)
                TH.start()            
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
            for p in jobs:
                p.join()
            if verbose>0:
                print("all threads are done")
            order=[]
            for ele in resultdict:
                order.append(ele[0])
                if len(np.where(np.asarray(ele[1])==False)[0])>0:
                    success=False                
            for i in range(threads):
                framess.append(resultdict[order.index(i)][3])
                timeline=[]
                for j in range(len(resultdict[order.index(i)][2])):
                    timeline.append([resultdict[order.index(i)][2][j],j,Psets[i],i])
                timesets.append(timeline)
                resultdict[order.index(i)]=[]
For faster browsing, not all history is shown. View entire blame