downloadversionIRdata.py 138 KB
Newer Older
Holger Niemann's avatar
Holger Niemann committed
1
2
3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
4
updated on Tue Aug 21 10:20:00 2018
5
last update on Fr Nov 23 15:37:00 2018
Holger Niemann's avatar
Holger Niemann committed
6

7
Version: 3.1.1
8
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
Holger Niemann's avatar
Holger Niemann committed
9
10
11
12
@author: holn
"""

import numpy as np
13
import IR_image_tools as IR_tools
14
15
16
17
18
from IR_config_constants import archivepath,portcamdict,camlimdict, \
                                valid_FOV_circle,valid_background_rectangle, \
                                TC_port,TC_channel,IRCamColdframes_fittingpath, \
                                exJet,portpathdict,project,project_ana, \
                                heatflux_requestlist_path#,exJet_trans
19
20
21
22
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
Holger Niemann's avatar
Holger Niemann committed
23
24
25
import datetime
import urllib
import json
26
27
from PIL import Image
from io import BytesIO
28
from os.path import join
29
import matplotlib.pyplot as plt
30
31
try:          
    import threading
32
    import archivedb as AKF_2
33
34
35
36
    fastDL=True
except Exception as E:
    print(E)
    fastDL=False
Holger Niemann's avatar
Holger Niemann committed
37

38

39
try:
40
    config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
41
42
43
44
45
    FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0]   
    del FOV
except:
    config_path=""

46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

# These two functions seem unused and will be deleted.
# They are redundant to 
# IR_tools.get_OP_by_time(time_ns=None, shot_no=None, program_str=None)
# which should be used instead.
# 
#def get_OP_by_program(program):
#    try:    
#        bla=int(program.split(".")[0])
#        time_ns=int(TimeToNs([bla // 10000,bla % 10000 // 100,bla % 10000 % 100],[10,0,0,0]))
#        return get_OP_by_time(time_ns)
#    except Exception as E:
#        print(E)
#        return None
#    
#def get_OP_by_time(time_ns):
#    dateOP=datetime.datetime.utcfromtimestamp(time_ns/1e9)
#    if dateOP.year==2017:
#        if dateOP.month>8 and dateOP.month<12:
#            return "OP1.2a"
#        elif dateOP.month==8 and dateOP.day>=28:
#            return "OP1.2a"
#        elif dateOP.month==12 and dateOP.day<8:
#            return "OP1.2a"
#        else:
#            return None        
#    elif dateOP.year==2018:
#        return "OP1.2b"
#    elif dateOP.year<=2016 and dateOP.year>=2015:
#        if (dateOP.year==2016 and dateOP.month<=3) or (dateOP.year==2015 and dateOP.month==12):
#            return "OP1.1"
#        else:
#            return None

80

81
def get_latest_version(stream,project="W7X",Test=False,t_from=None,t_to=None,program=None):
82
83
84
85
86
     """Find out latest version of given stream
     :param stream   url stream of interest
     :return int of version number of None if non-versioned or non-existing stream
     author: G. Schlisio, edit by holn
     """
87
88
89
90
     if Test:
         base="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
     else:
         base=archivepath    
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

    
     if program==None and t_from==None and t_to==None:         
         request =urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})            
     else:
         if program is not None:
             prog=get_program_from_PID(program)
             if prog[0]:
                 t_from=prog[1]['trigger']['1'][0]
                 t_to=prog[1]['trigger']['6'][0]
         elif t_from is not None and t_to==None:
             t_to=int(t_from+100e9)
         elif t_from==None and t_to is not None:
             t_from=int(t_to-10e9)
         request =urllib.request.Request(base +project+"/" + stream + "/_versions.json?from="+str(t_from)+"&upto="+str(t_to), headers={"Accept": "application/json"})
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
     try:
         response = urllib.request.urlopen(request)
         d = json.loads(response.read().decode('utf-8'))
     except urllib.error.HTTPError as ex:
         msg = ex.read()
         raise RuntimeError(msg)
     # detect unversioned or non-existing stream
     if d["versionInfo"] == []:
         return None
     versions = []
     for i in d['versionInfo']:
         versions.append(i['number'])

     return max(versions)

Holger Niemann's avatar
Holger Niemann committed
121
122
123
124
125
126
127
128
129
130
131
def TimeToNs(date,time):
    """
    TimeToNs(date,time)
    (list)  date    [year,month,day]
    (list)  time    [hours,minutes,seconds,microseconds]
    """ 
    date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3])
    div=date_time-datetime.datetime(1970,1,1,0,0,0)
    nsdate=div.total_seconds()*1e9
    return nsdate

132
def read_program(timestamp_start,timestamp_end=0,tol=60):
Holger Niemann's avatar
Holger Niemann committed
133
134
135
136
137
    """
    read_program()
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
    if timestamp_end!=0:
138
139
140
141
        timestamp_end = timestamp_start
    jsonurl = '{0}={1}&upto={2}'.format(program_url,
                                        int(timestamp_start-tol*1e9),
                                        int(timestamp_end+tol*1e9))
Holger Niemann's avatar
Holger Niemann committed
142
143
144
145
146
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw=res.read()
        res.close()
    except urllib.error.URLError as e:
147
        print('read_program: Error opening URL')
Holger Niemann's avatar
Holger Niemann committed
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
        print(e)
        return False,0
    else:
        prog_string=prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl=prog_list['programs'][0]
        return True, pl

def read_restdb_old(request_url):
    """
    Reads JSON data from W7-X REST API
    Returns: 
        valid: access ok
        t: numpy-array of time
        signal: numpy-array of requested signals
163
    by H. Thomsen
Holger Niemann's avatar
Holger Niemann committed
164
165
166
167
    """
    try:
        res = urllib.request.urlopen(request_url)
    except urllib.error.URLError as e:
168
        print('read_restdb_old: Error ', e)
Holger Niemann's avatar
Holger Niemann committed
169
170
171
172
173
174
175
176
177
178
        return False, 0, -1
    else:
        signal_raw=res.read()
        res.close()
        signal_string=signal_raw.decode(encoding='UTF-8')
        signal_list = json.loads(signal_string)
        signal0=np.array(signal_list['values'])
        t=np.array(signal_list['dimensions'])
        return True, t, signal0

179
def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
Holger Niemann's avatar
Holger Niemann committed
180
181
182
    """
    download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
        time in ns
183
    Have to swap 11, 21 until correction in the database
Holger Niemann's avatar
Holger Niemann committed
184
    """    
185
    OP=IR_tools.get_OP_by_time(time_ns=time)
186
187
188
189
    if port == 21:
        port = 11
    elif port == 11:
        port = 21
190
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
191
    if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
192
        query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
193
    elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
Holger Niemann's avatar
Holger Niemann committed
194
195
        query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
196
        print("download_LUT: Error! Camera unknown, stopping here.")
Holger Niemann's avatar
Holger Niemann committed
197
        raise Exception
198
    if version==0:
199
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM")
200
    if verbose>0:
201
        print("download_LUT: LUT V"+str(version)+" is used")
Holger Niemann's avatar
Holger Niemann committed
202
    #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
203
    LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
204
205
    if LUTpar[0]:
        LUTid=LUTpar[2][0]['structure'][query]
206
        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
207
        if LUTs[0]:
208
            LUTs=LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
Holger Niemann's avatar
Holger Niemann committed
209
210
211
212
            LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True,LUT
        else:
213
            print("download_LUT: Warning! unable to download the LUTs")
Holger Niemann's avatar
Holger Niemann committed
214
215
216
217
            del LUTpar, LUTs
            return False,0
    else:
        del LUTpar
218
        print("download_LUT: Warning! unable to find LUTs, check your request")
Holger Niemann's avatar
Holger Niemann committed
219
220
        return False,0

221
222
223
224
225
226
227
228

def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
    
    Read LUT V3 from local files for testing new calibration.
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
229
    OP = IR_tools.get_OP_by_time(time_ns=this_time)
230
231
232
233
234
235
236
237
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
        with open(LUT_dir+"\\"+filename) as data_file:  
            jfile=json.load(data_file)
        LUT=np.array(jfile['LUT']).swapaxes(0,1)
        if verbose>0:
238
            print("read_LUT_from_file: succesfully loaded V3 LUT from local directory")
239
240
        return True, LUT
    except Exception as E:
241
        print("read_LUT_from_file: ERROR in loading V3 LUTs",E)
242
243
        return False, []
    
244
def download_NUC_by_program(port,program,exposure,version=0):
245
    prog=get_program_from_PID(program)
246
247
248
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
Holger Niemann's avatar
Holger Niemann committed
249
250
251
252
253
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_NUC_by_times(port,starttime,stoptime,exposure,version)
    else:
254
#    except:
255
        print("download_NUC_by_program: cannot find the program")
Holger Niemann's avatar
Holger Niemann committed
256
257
        return False,0,0

258
def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
259
    project="W7X"
260
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
261
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
Holger Niemann's avatar
Holger Niemann committed
262
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
263
    if version==0:
264
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
265
266
267
268
269
270
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
271
        print('download_NUC_by_times: Error! ', e)
Holger Niemann's avatar
Holger Niemann committed
272
273
274
275
276
277
278
279
280
281
282
        goon=False
    if goon:        
        n=0
        nuctimes=[0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp']==exposure:
                nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i=NUCpar['structure']['offset_index']                
            n+=1 
        if nuctimes[0]!=0:
283
            NUC=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
284
285
286
287
            if NUC[0]:
                images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)                    
                return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
            else:
288
                print("download_NUC_by_times: NUC image for requested exposure time not found")
Holger Niemann's avatar
Holger Niemann committed
289
290
                return False, 0, 0
        else:
291
            print("download_NUC_by_times: NUC image for requested exposure time not found")
Holger Niemann's avatar
Holger Niemann committed
292
293
294
            return False,0,0
    else:
        return False,0,0
295

296
def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
297
    prog=get_program_from_PID(program)
298
299
300
301
302
303
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
304
        return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose-1)
305
306
    else:
#    except:
307
        print("get_NUC_by_program: Error! cannot find the program")
308
309
        return False,0,0
    
310
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
311
312
313
314
315
    """
    Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera
    in a given port and time interval. Depending on time and camera, reconstruct
    teh cold frame and/or compute the gain,offset from the hot and cold frame.
    """
316
317
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if OP=="OP1.2a":
318
319
320
321
        t1=endtime
        t0=starttime
        prog=read_program(t1)
        program=prog[1]['id']
322
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):
323
324
            if verbose>0:
                print("get_NUC_by_times: rebuilding coldframe")
325
326
327
328
329
330
331
332
            #use any way the rebuilt coldframe.
            sT = get_sensor_temp_by_program(port, program)[2][0]
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
333
                raise Exception("get_NUC_by_times: unable to download reference frames")
334
335
336
337
            filestring = 'AEF' + str(port) + '_et' + str(t_exp)
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
338
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
339
340
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
341
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)
342
343
344
345
346
347
348
349
350
351
            gain_error=0
            offset_error=0        
            return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        elif (port == 11) or (port == 21):
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
352
                raise Exception("get_NUC_by_times: unable to download reference frames")
353
354
            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
            if NUC_DL[0]==False:
355
                raise Exception("get_NUC_by_times: NUC was not found")
356
357
            else:
                ci = NUC_DL[1][2]
358
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1)
359
360
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
361
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)                                
362
363
            gain_error=0
            offset_error=0
364
            return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
365
366
367
368
369
370
371
372
373
374
        else:
            return download_NUC_by_times(port,starttime,endtime,t_exp)
    elif OP=="OP1.2b":
        gain_error=0
        offset_error=0
        gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
        if gotit:
            coldref=hotcold[1]
            hotref=hotcold[0]
        else:
375
            raise Exception("get_NUC_by_times: unable to download reference frames")
376
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
377
        exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0,verbose=verbose-1)
378
379
380
381
382
383
384
        if exist:
            cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
            for ele in coldframes:
                cold+=ele                
            cold=np.asarray(cold/len(coldframes),dtype=np.uint16)        
        else:
            return False,[0],[0]
385
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
386
        badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)
387
388
        return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
    else:
389
        raise Exception("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP")
390

391
def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0,verbose=0):
392
393
394
395
    """
    download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
    """
Holger Niemann's avatar
Holger Niemann committed
396
    gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(endtime-20e6))
397
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
398
399
400
    if gotit:
        expinds=np.where(texp_t==t_exp)[0]
        if len(expinds)==0:
401
            print("download_calibration_raw_files_by_time: Error! cannot find the exposure time in the given data")
402
403
            return False,[0],[0]                               
    else:
404
        print("download_calibration_raw_files_by_time: Error! exposure time not found")
405
        return False,[0],[0]
Holger Niemann's avatar
Holger Niemann committed
406
    gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(endtime-20e6))
407
    if gotitf:
Holger Niemann's avatar
Holger Niemann committed
408
        typiinds=np.where(values_f[expinds]==frametype)[0]
409
        if len(typiinds)>0:
Holger Niemann's avatar
Holger Niemann committed
410
411
412
            ref_t=[np.min(timef[expinds][typiinds]),np.max(timef[expinds][typiinds])]
#            print((ref_t[1]-ref_t[0])/1e9)
#            print(len(timef),len(timef[expinds]),len(timef[expinds][typiinds]))
413
        else:#okay the early data stuff or strange stuff
414
415
            if verbose>0:
                print("download_calibration_raw_files_by_time: frame type was not identified assuming that the first part is the cold and the second one the background")
416
417
418
419
420
            frametimes=time_t[expinds]
            diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
            turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
            if frametype==0:
                ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
Holger Niemann's avatar
Holger Niemann committed
421
                print((ref_t[1]-ref_t[0])/1e9)
422
423
424
            elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
                ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
Holger Niemann's avatar
Holger Niemann committed
425
                print((ref_t[1]-ref_t[0])/1e9)
426
            else:
427
                raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
428
    else:
429
        print("download_calibration_raw_files_by_time: Error! frametype not found")
430
431
432
        return False,[0],[0]
    t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
    t1date=t1date.isoformat()
Holger Niemann's avatar
Holger Niemann committed
433
    t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
434
435
    t0date=t0date.isoformat()
    if version==0:
436
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=endtime)
437
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
438
    timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
Holger Niemann's avatar
Holger Niemann committed
439
    t_offset=timest0[-1][0]-time_t[0]
440
    if t_offset>10e6:
441
442
        if verbose>0:
            print("download_calibration_raw_files_by_time: time offset detected, try to correct this, offset is",t_offset)
443
444
445
    else:
        t_offset=0
#            print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
446
    return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset,verbose=verbose-1)
447
448
        
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
449
450
451
452
    """
    Loads the most recent hot and cold calibration frames for a starttime.
    Uses first calibration frames if time is not defined.
    """
453
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
454
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
455
456
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
457
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime)
458
459
460
461
462
463
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
464
        print('download_hot_cold_reference_by_times: Error! ',e)
465
466
467
468
469
470
471
        goon=False
    if goon:        
        COLDtime=0
        HOTtime=0
        try:
            COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
        except:
472
            print("download_hot_cold_reference_by_times: cold image for requested exposure time not found")
473
474
475
        try:
            HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
        except:
476
            print("download_hot_cold_reference_by_times: hot image for requested exposure time not found")
477
478
        images=[]
        if HOTtime!=0:
479
            HOT=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
480
481
482
            if HOT[0]:
                images.append(HOT[2])
        if COLDtime!=0:
483
            COLD=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
484
485
486
487
488
489
            if COLD[0]:
                images.append(COLD[2])
                
        if HOT[0] and COLD[0] and len(images)==2:
            return True,images,['hot','cold']
        else:
490
            print("download_hot_cold_reference_by_times: Error! hot and cold image for requested exposure time not found")
491
492
493
            return False,0,0
    else:
        return False,0,0
Holger Niemann's avatar
Holger Niemann committed
494
    
495
def download_background_by_program(port,program,exposure,version=0):
496
497
498
    '''
    returned exist,time,frame
    '''
499
    prog=get_program_from_PID(program)
500
501
502
503
504
505
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
506
        return download_background_by_times(port,starttime,stoptime,exposure,version)
507
    else:#except:
508
        print("download_background_by_program: Error! cannot find the program {0}".format(program))
Holger Niemann's avatar
Holger Niemann committed
509
510
        return False,0,0
    
511
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
512
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
513
    stream=portpathdict[OP]["AEF"+str(port)]+"background_"
514
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
515
516
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
517
518
519
520
521
522
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
523
        print('download_background_by_times: Error! ',e)
Holger Niemann's avatar
Holger Niemann committed
524
525
526
527
528
529
        goon=False
    if goon: 
        n=0
        backtimes=[0]
        for backpar in signal_list['values']:
            if backpar['structure']['Texp']==exposure:
530
                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
531
532
533
534
535
536
                    if backpar['structure']['filter']==camera_filter:
                        backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
                else:
                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
            n+=1
        if backtimes[0]!=0:
537
            backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
538
539
540
            if backdat[0]:                                    
                return backdat#[True,backdat[2]]
            else:
541
                print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
Holger Niemann's avatar
Holger Niemann committed
542
543
                return False,0,0
        else:
544
            print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
Holger Niemann's avatar
Holger Niemann committed
545
546
547
548
            return False,0,0
    else:
        return False,0,0

549
550
551
552
553
def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False,verbose=0):
    """
    OP1.2b function
    """
    exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version,verbose=verbose-1)
554
    camera=portpathdict["OP1.2b"]["AEF"+str(port)]    
555
556
557
558
559
560
561
562
563
    background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
    for frame in backgroundframes:
        background+=frame
    background=np.asarray(background/len(backgroundframes),dtype=np.uint16)
    if plot_it:
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image unnuced")
    if not camera.split("_")[0]=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
564
        background=apply_NUC([background],gain,offset)[0]
565
566
567
568
569
570
    if plot_it:        
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image nuced")
    return exist,btime[0],background

571
572
573
def download_raw_images_by_program(port,program,time_window=0,version=0,verbose=0):
    """
    """
574
575
576
577
578
579
580
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
581
582
583
584
585
        if type(time_window)==list or type(time_window)==np.ndarray:
            if len(time_window)>2 or time_window[0]>1000:
                raise Exception("download_raw_images_by_program: invalid input for the time_window!")
            tstart=time_window[0]
            tstop=time_window[1]
Holger Niemann's avatar
Holger Niemann committed
586
            if tstop<tstart:
587
                raise Exception("download_raw_images_by_program: endtime before starttime")
588
            return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
589
        else:
590
            if time_window==0:
591
                return download_raw_images_by_times(port,starttime,stoptime,version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
592
            else:
593
                return download_raw_images_by_times(port,starttime,int(starttime+time_window*1e9),version,verbose=verbose-1)
594
    except:
595
        print("download_raw_images_by_program: Error! cannot find the program")
Holger Niemann's avatar
Holger Niemann committed
596
597
        return False,0,0

598
def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
599
600
    """
    """
601
602
    OP=IR_tools.get_OP_by_time(time_ns=starttime)    
    if version==0:
603
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime)
604
605
606
607
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"  
    return download_images_by_times(larchivepath,starttime=starttime,stoptime=stoptime,version=version,intervalSize=intervalSize,verbose=verbose-1)
    
def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
608
609
    """
    """
610
611
612
613
    if '_raw' in larchivepath:
        typo=np.uint16          
    else:
        typo=np.float32
614
615
    if (stoptime-starttime)/intervalSize>1:        
        nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
616
        if verbose>0:
617
            print("download_images_by_times: time_window to large, splitting {0}s interval into {1} smaller fractions".format((stoptime-starttime)/1e9,nrinterv))
Holger Niemann's avatar
Holger Niemann committed
618
        intervalls=[]
619
        for i in range(nrinterv):
620
            intervalls.append(int(starttime-10+i*intervalSize))
Holger Niemann's avatar
Holger Niemann committed
621
        intervalls.append(stoptime)
622
623
624
        time=-1
        allimages=0
        success=False
625
        for i in range(nrinterv):
Holger Niemann's avatar
Holger Niemann committed
626
            try:
Holger Niemann's avatar
Holger Niemann committed
627
                res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1]))
Holger Niemann's avatar
Holger Niemann committed
628
                signal_list = json.loads(res.read().decode('utf-8'))
629
630
                res.close()                
                images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
631
#                signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
632
633
634
                t=np.array(signal_list['dimensions'])
                del signal_list
                if i==0:
635
                    allimages=images.copy()
Holger Niemann's avatar
Holger Niemann committed
636
637
                    time=t
                else:
638
                    allimages=allimages+images#np.append(ressignal,signal,axis=0)
Holger Niemann's avatar
Holger Niemann committed
639
                    time=np.append(time,t)
640
                del images,t
641
                success=True
Holger Niemann's avatar
Holger Niemann committed
642
            except urllib.error.URLError as e:
643
                print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e))                
644
        return success,time,allimages
Holger Niemann's avatar
Holger Niemann committed
645
646
647
648
649
    else:
        try:
            res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
650
            images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
651
#            signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
652
653
            t=np.array(signal_list['dimensions'])
            del signal_list
654
            return True, t, images
Holger Niemann's avatar
Holger Niemann committed
655
        except urllib.error.URLError as e:
656
            print('download_images_by_times: Error! ',e)
Holger Niemann's avatar
Holger Niemann committed
657
            return False, 0,-1
658

659
if fastDL:
660
661
662
    def download_raw_images_by_program_via_png(port,program,time_window=0,version=0,threads=1,verbose=0):
        """
        """
663
        prog=get_program_from_PID(program)
664
665
666
#        try:
#            t_program = AKF_2.get_program_from_to(program)
#            prog =AKF_2.get_program_list(t_program[0], t_program[1])
667
        if not prog[0]:
668
669
            print("download_raw_images_by_program_via_png: Error! program not found, cannot download the data")
            return False,[0],[0]
670
        else:
671
672
            starttime=prog[1]['trigger']['1'][0]
            stoptime=prog[1]['trigger']['6'][0]
673
            OP=IR_tools.get_OP_by_time(time_ns=starttime)
674
            if version==0:
675
                version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime)
676
            larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
677
678
679
            return download_images_by_time_via_png(larchivepath,starttime,stoptime,
                                                   time_window=time_window,threads=threads,
                                                   version=version,verbose=verbose-1)
680
            
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
    def download_images_by_time_via_png(larchivepath,starttime,stoptime,time_window=0,
                                        threads=1,version=0,verbose=0):
        """
        """
        success=True            
        if type(time_window)==list:
            tstart=time_window[0]
            tstop=time_window[1]
            if tstop<tstart:
                raise Exception("download_images_by_time_via_png: endtime before starttime")
            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
            stdate=stdate.isoformat()
            enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
            enddate=enddate.isoformat()
        else:
            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
            stdate=stdate.isoformat()
            if time_window==0:
                enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
700
701
                enddate=enddate.isoformat()
            else:
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window)  
                enddate=enddate.isoformat()
            #"2017-11-15 08:00:00"
        try:
            times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
        except Exception as E:
            print('download_images_by_time_via_png: Error loading times from ',larchivepath)
            raise Warning(E)
            return False,[0],[0]            
        time=[]
        images=[]
        lnt=len(times)
        if threads==1:
            for i in range(lnt):
                ele=times[lnt-1-i]
                imag=download_last_raw_image_by_time(larchivepath,ele[0]-10,ele[0]+10)
                if imag[0]:
                    time.append(ele[0])
                    images.append(np.array(imag[1],dtype=np.uint16))
Holger Niemann's avatar
Holger Niemann committed
721
                else:
722
723
724
725
726
727
728
729
730
731
732
733
734
735
                    success=False
            return success,np.array(time),images
        else:
            tim=[]
            for i in range(lnt):
                tim.append(times[lnt-1-i][0])
            intervalls=[]
            intervalSize=int(lnt/threads)
            for i in range(threads):
                intervalls.append(int(i*intervalSize))
            intervalls.append(lnt)
            jobs = []
            resultdict = []
            for i in range(threads):
736
                if verbose>0:
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
                    print("download_images_by_time_via_png: Start Thread ",i+1)                
                TH=download_images_thread(i,larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,version)
                jobs.append(TH)
                TH.start()
            for p in jobs:
                p.join()
            if verbose>0:
                print("download_images_by_time_via_png: all threads are done")
            order=[]
            for ele in resultdict:
                order.append(ele[0])
                if len(np.where(np.asarray(ele[1])==False)[0])>0:
                    success=False                
            times=[]#np.array(resultdict[order.index(0)][2])
            images=[]#np.array(resultdict[order.index(0)][3])
            for i in range(threads):
                images=images+resultdict[order.index(i)][3]
                times=times+resultdict[order.index(i)][2]
                resultdict[order.index(i)]=[]
            del resultdict
            return success,np.array(times),images
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
      
    class download_images_thread(threading.Thread):
        def __init__(self, threadID, larchivepath,times,resultlist,version=0):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            self.resultlist.append([self.threadID,successes,time,images])
780
    
781
    class convert_raw_to_temp_thread(threading.Thread):
782
783
784
        def __init__(self,larchivepath,times,resultlist,threadID,version=0,background=0,
                     LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,
                     give_ERROR=False,FOV=0,badpixels=[],verbose=0):
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
801
            self.verbose=verbose
802
803
804
805
806
807
808
809
810
811
812
813
814
        def run(self):
            images=[]
            time=[]
            successes=[]        
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            if self.give_ERROR:
815
                sucess,images,error_images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR,verbose=self.verbose-1)
816
            else:
817
                success,images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR,verbose=self.verbose-1)
818
819
820
            
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
821
822
823
            if self.verbose>0:
                print(datetime.datetime.now(),"convert_raw_to_temp_thread: correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
824
            self.resultlist.append([self.threadID,successes,time,images])
825
    
826
    class NUC_raw_thread(threading.Thread):
827
828
829
        def __init__ (self,larchivepath,times,resultlist,threadID,version=0,background=0,
                      LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,
                      give_ERROR=False,FOV=0,badpixels=[]):
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
858
859
            if self.verbose>0:
                print(datetime.datetime.now(),"NUC_raw_thread: applying NUC")
860
861
            if self.give_ERROR:
                images,error_images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
862
            else:
863
864
865
866
                images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
            
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
867
868
869
            if self.verbose>0:
                print(datetime.datetime.now(),"NUC_raw_thread: correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
870
            self.resultlist.append([self.threadID,successes,time,images]) 
871

872
873
874
def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,verbose=0):
    """
    """
875
    larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
876
877
878
    if type(time_window)==list:
        tstart=time_window[0]
        tstop=time_window[1]
879
        if tstop<tstart:
880
            raise Exception("download_raw_FLIR_images_via_png: endtime before starttime")
881
882
883
884
885
886
887
        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
        stdate=stdate.isoformat()
        enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)  
        enddate=enddate.isoformat()
    else:
        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
        stdate=stdate.isoformat()
888
        if time_window==0:
889
890
891
            enddate=datetime.datetime.utcfromtimestamp(t6/1e9)        
            enddate=enddate.isoformat()
        else:
892
            enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_window)  
893
894
895
896
897
898
899
900
901
902
            enddate=enddate.isoformat()
        #"2017-11-15 08:00:00"
    typ1_version=get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM")
    if typ1_version==None:
        mode=1
    else:
        mode=2
    if mode==1:
        ### okay the data is stored probably as different sets (so recorded with FLIR software)
        timess,versions=get_FLIR_Pset_times(stdate,enddate,version)
903
        if verbose>0:
904
            print("download_raw_FLIR_images_via_png: found {0} frames in the different sets".format([np.shape(x) for x in timess]))
905
906
907
        ### we not know the times for all presets for the program, now we have to download them, if times is not empty
        exist,time,frames,tsets=download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=verbose-1)            
    elif mode==2:##the data is stored only in one datastream, at least this datastream exist
908
        raise Exception("download_raw_FLIR_images_via_png: not implemented")
909
910
        #download data and downlaod LUT
        
911
912
913
914
915
916
917
918
919
920
921
922
923
#        time=[]
#        images=[]
#        lnt=len(times)
#        if threads==1:
#            for i in range(lnt):
#                ele=times[lnt-1-i]
#                imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
#                if imag[0]:
#                    time.append(ele[0])
#                    images.append(np.array(imag[1],dtype=np.uint16))
#                else:
#                    success=False
#            return success,np.array(time),images
924
    else:
925
        raise Exception("download_raw_FLIR_images_via_png: cannot identifie the way the searched data is stored!")                        
926
927
928

    return exist,time,frames,mode,tsets,versions

929
def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0):
930
931
    """
    """
932
933
934
935
936
937
938
939
940
941
    larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
    if version==0:
        versions=[]
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS0_DATASTREAM"))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS1_DATASTREAM"))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS2_DATASTREAM"))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS3_DATASTREAM"))
    else:
        versions=[version,version,version,version]
    timess=[]
942
943
944
    if version==0:
        if verbose>0:
            print('get_FLIR_Pset_times: no version specified search for data in all versions')
945
946
947
948
949
950
        counter=-1
        for version0 in versions:
            counter+=1
            if version0>0:
                found=False
                while (version0>0 and not found):                    
951
952
953
                    try:
                        url = '{0}_PS{1}_DATASTREAM/V{2}/0/raw'.format(larchivepath,counter,version0)
                        times_0=AKF_2.get_time_intervals(url,stdate.replace("T"," "),enddate.replace("T"," "))#
954
955
                        found=True
                    except Exception as E:
956
                        print('get_FLIR_Pset_times: Error querrying {0}\n{1}'.format(url,E))
957
958
959
960
961
962
963
964
965
966
                        found=False
                        times_0=[]
                        version0=version0-1
                timess.append(times_0)
            else:
                timess.append([])
    else:
        counter=-1
        for version0 in versions:
            counter+=1
967
968
969
            try:
                url = '{0}_PS{1}_DATASTREAM/V{2}/0/raw'.format(larchivepath,counter,version0)
                times_0=AKF_2.get_time_intervals(url,stdate.replace("T"," "),enddate.replace("T"," "))#
970
            except Exception as E:
971
                print('get_FLIR_Pset_times: Error querrying {0}\n{1}'.format(url,E))
972
973
974
975
976
                times_0=[]
            timess.append(times_0)
    return timess,versions

def download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=0):
977
978
    """
    """
979
980
981
982
983
984
985
    if len(timess[0])==len(timess[1]) and len(timess[2])==len(timess[3]) and len(timess[0])==0:
            exist=False
            time=[]
            frames=[]
            tsets=[]
    else:#timess is not empty, there is data in at least one Pset
        if verbose>0:
986
            print(datetime.datetime.now(),"download_FLIR_Psets: data found, start the download")
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
        exist=True
        timesets=[]
        framess=[]
        success=True
        Psets=[]
        if threads==1:            
            counter=-1
            setn=-1
            for times in timess:                    
                setn+=1
                if times!=[]:
                    counter+=1
                    Psets.append(counter)
                    framess.append([])
For faster browsing, not all history is shown. View entire blame