downloadversionIRdata.py 154 KB
Newer Older
Holger Niemann's avatar
Holger Niemann committed
1
2
3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
4
updated on Tue Aug 21 10:20:00 2018
5
last update on Fr Nov 23 15:37:00 2018
Holger Niemann's avatar
Holger Niemann committed
6

7
Version: 3.2.2
8
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
Holger Niemann's avatar
Holger Niemann committed
9
10
11
12
@author: holn
"""

import numpy as np
13
import IR_image_tools as IR_tools
14
15
16
17
18
from IR_config_constants import archivepath,portcamdict,camlimdict, \
                                valid_FOV_circle,valid_background_rectangle, \
                                TC_port,TC_channel,IRCamColdframes_fittingpath, \
                                exJet,portpathdict,project,project_ana, \
                                heatflux_requestlist_path#,exJet_trans
19
20
21
22
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
Holger Niemann's avatar
Holger Niemann committed
23
24
25
import datetime
import urllib
import json
26
27
from PIL import Image
from io import BytesIO
28
from os.path import join
29
import matplotlib.pyplot as plt
30
31
try:          
    import threading
32
    import archivedb as AKF_2
33
34
35
36
    fastDL=True
except Exception as E:
    print(E)
    fastDL=False
Holger Niemann's avatar
Holger Niemann committed
37

38

39
try:
40
    config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
41
42
43
44
45
    FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0]   
    del FOV
except:
    config_path=""

46

47
def get_latest_version(stream,project="W7X",Test=False,t_from=None,t_to=None,program=None):
48
    
49
50
51
52
53
     """Find out latest version of given stream
     :param stream   url stream of interest
     :return int of version number of None if non-versioned or non-existing stream
     author: G. Schlisio, edit by holn
     """
54
55
56
57
     if Test:
         base="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
     else:
         base=archivepath    
58
59
60
61
62
63
64
65

    
     if program==None and t_from==None and t_to==None:         
         request =urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})            
     else:
         if program is not None:
             prog=get_program_from_PID(program)
             if prog[0]:
66
                 t_from=prog[1]['trigger']['0'][0]
67
68
69
70
71
72
                 t_to=prog[1]['trigger']['6'][0]
         elif t_from is not None and t_to==None:
             t_to=int(t_from+100e9)
         elif t_from==None and t_to is not None:
             t_from=int(t_to-10e9)
         request =urllib.request.Request(base +project+"/" + stream + "/_versions.json?from="+str(t_from)+"&upto="+str(t_to), headers={"Accept": "application/json"})
73
74
75
76
77
78
     try:
         response = urllib.request.urlopen(request)
         d = json.loads(response.read().decode('utf-8'))
     except urllib.error.HTTPError as ex:
         msg = ex.read()
         raise RuntimeError(msg)
79
80
     except Exception as E:
         raise RuntimeError(E)
81
82
     else:
         response.close()
83
84
85
86
87
88
89
90
91
     # detect unversioned or non-existing stream
     if d["versionInfo"] == []:
         return None
     versions = []
     for i in d['versionInfo']:
         versions.append(i['number'])

     return max(versions)

Holger Niemann's avatar
Holger Niemann committed
92
93
94
95
96
97
98
99
100
101
102
def TimeToNs(date,time):
    """
    TimeToNs(date,time)
    (list)  date    [year,month,day]
    (list)  time    [hours,minutes,seconds,microseconds]
    """ 
    date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3])
    div=date_time-datetime.datetime(1970,1,1,0,0,0)
    nsdate=div.total_seconds()*1e9
    return nsdate

103
def read_program(timestamp_start,timestamp_end=0,tol=60):
Holger Niemann's avatar
Holger Niemann committed
104
105
106
107
    """
    read_program()
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
108
    if timestamp_end==0:
109
110
111
112
        timestamp_end = timestamp_start
    jsonurl = '{0}={1}&upto={2}'.format(program_url,
                                        int(timestamp_start-tol*1e9),
                                        int(timestamp_end+tol*1e9))
Holger Niemann's avatar
Holger Niemann committed
113
114
115
116
117
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw=res.read()
        res.close()
    except urllib.error.URLError as e:
118
        print('read_program: Error opening URL')
Holger Niemann's avatar
Holger Niemann committed
119
120
        print(e)
        return False,0
121
122
123
    except Exception as e:
        print('read_program: Error opening URL',e)
        return False,0
Holger Niemann's avatar
Holger Niemann committed
124
125
126
127
128
129
    else:
        prog_string=prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl=prog_list['programs'][0]
        return True, pl

130

Holger Niemann's avatar
Holger Niemann committed
131

132
def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
Holger Niemann's avatar
Holger Niemann committed
133
134
135
    """
    download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
        time in ns
136
    Have to swap 11, 21 until correction in the database
Holger Niemann's avatar
Holger Niemann committed
137
    """    
138
    OP=IR_tools.get_OP_by_time(time_ns=time)
139
140
141
142
    if port == 21:
        port = 11
    elif port == 11:
        port = 21
143
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
144
    if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
145
        query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
146
    elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
Holger Niemann's avatar
Holger Niemann committed
147
148
        query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
149
        print("download_LUT: Error! Camera unknown, stopping here.")
Holger Niemann's avatar
Holger Niemann committed
150
        raise Exception
151
    if version==0:
152
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM")
153
    if verbose>0:
154
        print("download_LUT: LUT V"+str(version)+" is used")
Holger Niemann's avatar
Holger Niemann committed
155
    #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
156
    LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
157
158
    if LUTpar[0]:
        LUTid=LUTpar[2][0]['structure'][query]
159
        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
160
        if LUTs[0]:
161
            LUTs=LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
Holger Niemann's avatar
Holger Niemann committed
162
163
164
165
            LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True,LUT
        else:
166
            print("download_LUT: Warning! unable to download the LUTs")
Holger Niemann's avatar
Holger Niemann committed
167
168
169
170
            del LUTpar, LUTs
            return False,0
    else:
        del LUTpar
171
        print("download_LUT: Warning! unable to find LUTs, check your request")
Holger Niemann's avatar
Holger Niemann committed
172
173
        return False,0

174
175
176
177
178
179
180
181

def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
    
    Read LUT V3 from local files for testing new calibration.
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
182
    OP = IR_tools.get_OP_by_time(time_ns=this_time)
183
184
185
186
187
188
189
190
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
        with open(LUT_dir+"\\"+filename) as data_file:  
            jfile=json.load(data_file)
        LUT=np.array(jfile['LUT']).swapaxes(0,1)
        if verbose>0:
191
            print("read_LUT_from_file: succesfully loaded V3 LUT from local directory")
192
193
        return True, LUT
    except Exception as E:
194
        print("read_LUT_from_file: ERROR in loading V3 LUTs",E)
195
196
        return False, []
    
197
def download_NUC_by_program(port,program,exposure,version=0):
198
    prog=get_program_from_PID(program)
199
200
201
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
Holger Niemann's avatar
Holger Niemann committed
202
203
204
205
206
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_NUC_by_times(port,starttime,stoptime,exposure,version)
    else:
207
#    except:
208
        print("download_NUC_by_program: cannot find the program")
Holger Niemann's avatar
Holger Niemann committed
209
210
        return False,0,0

211
def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
212
    project="W7X"
213
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
214
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
Holger Niemann's avatar
Holger Niemann committed
215
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
216
    if version==0:
217
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
218
219
220
221
222
223
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
224
        print('download_NUC_by_times: Error! ', e)
Holger Niemann's avatar
Holger Niemann committed
225
        goon=False
226
227
228
    except Exception as e:
        print('download_NUC_by_times: Error! ', e)
        goon=False
Holger Niemann's avatar
Holger Niemann committed
229
230
231
232
233
234
235
236
237
238
    if goon:        
        n=0
        nuctimes=[0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp']==exposure:
                nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i=NUCpar['structure']['offset_index']                
            n+=1 
        if nuctimes[0]!=0:
239
            NUC=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
240
241
242
243
            if NUC[0]:
                images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)                    
                return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
            else:
244
                print("download_NUC_by_times: NUC image for requested exposure time not found")
Holger Niemann's avatar
Holger Niemann committed
245
246
                return False, 0, 0
        else:
247
            print("download_NUC_by_times: NUC image for requested exposure time not found")
Holger Niemann's avatar
Holger Niemann committed
248
249
250
            return False,0,0
    else:
        return False,0,0
251

252
def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
253
    prog=get_program_from_PID(program)
254
255
256
257
258
259
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
260
        return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose-1)
261
262
    else:
#    except:
263
        print("get_NUC_by_program: Error! cannot find the program")
264
265
        return False,0,0
    
266
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
267
268
269
270
271
    """
    Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera
    in a given port and time interval. Depending on time and camera, reconstruct
    teh cold frame and/or compute the gain,offset from the hot and cold frame.
    """
272
273
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if OP=="OP1.2a":
274
275
276
277
        t1=endtime
        t0=starttime
        prog=read_program(t1)
        program=prog[1]['id']
278
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):
279
280
            if verbose>0:
                print("get_NUC_by_times: rebuilding coldframe")
281
282
283
284
285
286
287
288
            #use any way the rebuilt coldframe.
            sT = get_sensor_temp_by_program(port, program)[2][0]
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
289
                raise Exception("get_NUC_by_times: unable to download reference frames")
290
291
292
293
            filestring = 'AEF' + str(port) + '_et' + str(t_exp)
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
Peter Drewelow's avatar
Peter Drewelow committed
294
295
            if verbose>0:
                print('get_NUC_by_times: calculate gain, offset and bad pixels')
296
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
297
298
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
299
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)
300
301
302
303
            gain_error=0
            offset_error=0        
            return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        elif (port == 11) or (port == 21):
Peter Drewelow's avatar
Peter Drewelow committed
304
305
            if verbose>0:
                print('get_NUC_by_times: downloading NUC')
306
307
308
309
310
311
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
312
                raise Exception("get_NUC_by_times: unable to download reference frames")
313
314
            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
            if NUC_DL[0]==False:
315
                raise Exception("get_NUC_by_times: NUC was not found")
316
317
            else:
                ci = NUC_DL[1][2]
318
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1)
319
320
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
321
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)                                
322
323
            gain_error=0
            offset_error=0
324
            return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
325
        else:
Peter Drewelow's avatar
Peter Drewelow committed
326
327
            if verbose>0:
                print('get_NUC_by_times: downloading NUC')
328
329
330
331
332
333
334
335
336
            return download_NUC_by_times(port,starttime,endtime,t_exp)
    elif OP=="OP1.2b":
        gain_error=0
        offset_error=0
        gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
        if gotit:
            coldref=hotcold[1]
            hotref=hotcold[0]
        else:
337
            raise Exception("get_NUC_by_times: unable to download reference frames")
338
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
339
        exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0,verbose=verbose-1)
340
341
342
343
344
345
346
        if exist:
            cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
            for ele in coldframes:
                cold+=ele                
            cold=np.asarray(cold/len(coldframes),dtype=np.uint16)        
        else:
            return False,[0],[0]
347
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
348
        badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)
349
350
        return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
    else:
351
        raise Exception("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP")
352

353
def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0,verbose=0):
354
355
356
357
    """
    download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
    """
Holger Niemann's avatar
Holger Niemann committed
358
    gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(endtime-20e6))
359
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
360
361
362
    if gotit:
        expinds=np.where(texp_t==t_exp)[0]
        if len(expinds)==0:
363
            print("download_calibration_raw_files_by_time: Error! cannot find the exposure time in the given data")
364
365
            return False,[0],[0]                               
    else:
366
        print("download_calibration_raw_files_by_time: Error! exposure time not found")
367
        return False,[0],[0]
Holger Niemann's avatar
Holger Niemann committed
368
    gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(endtime-20e6))
369
    if gotitf:
Holger Niemann's avatar
Holger Niemann committed
370
        typiinds=np.where(values_f[expinds]==frametype)[0]
371
        if len(typiinds)>0:
Holger Niemann's avatar
Holger Niemann committed
372
373
374
            ref_t=[np.min(timef[expinds][typiinds]),np.max(timef[expinds][typiinds])]
#            print((ref_t[1]-ref_t[0])/1e9)
#            print(len(timef),len(timef[expinds]),len(timef[expinds][typiinds]))
375
        else:#okay the early data stuff or strange stuff
376
377
            if verbose>0:
                print("download_calibration_raw_files_by_time: frame type was not identified assuming that the first part is the cold and the second one the background")
378
379
380
381
382
            frametimes=time_t[expinds]
            diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
            turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
            if frametype==0:
                ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
Holger Niemann's avatar
Holger Niemann committed
383
                print((ref_t[1]-ref_t[0])/1e9)
384
385
386
            elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
                ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
Holger Niemann's avatar
Holger Niemann committed
387
                print((ref_t[1]-ref_t[0])/1e9)
388
            else:
389
                raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
390
    else:
391
        print("download_calibration_raw_files_by_time: Error! frametype not found")
392
393
394
395
396
397
398
399
400
401
402
403
404
        frametimes=time_t[expinds]
        diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
        turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
        if frametype==0:
            ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
            print((ref_t[1]-ref_t[0])/1e9)
        elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
            ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
            print((ref_t[1]-ref_t[0])/1e9)
        else:
            raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
#        return False,[0],[0]
405
406
    t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
    t1date=t1date.isoformat()
Holger Niemann's avatar
Holger Niemann committed
407
    t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
408
409
    t0date=t0date.isoformat()
    if version==0:
410
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=endtime)
411
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
412
    timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
Holger Niemann's avatar
Holger Niemann committed
413
    t_offset=timest0[-1][0]-time_t[0]
414
    if t_offset>10e6:
415
416
        if verbose>0:
            print("download_calibration_raw_files_by_time: time offset detected, try to correct this, offset is",t_offset)
417
418
419
    else:
        t_offset=0
#            print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
420
    return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset,verbose=verbose-1)
421
422
        
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
423
424
425
426
    """
    Loads the most recent hot and cold calibration frames for a starttime.
    Uses first calibration frames if time is not defined.
    """
427
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
428
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
429
430
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
431
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime)
432
433
434
435
436
437
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
438
        print('download_hot_cold_reference_by_times: Error! ',e)
439
        goon=False
440
441
442
    except Exception as e:
        print('download_hot_cold_reference_by_times: Error! ',e)
        goon=False
443
444
445
446
447
448
    if goon:        
        COLDtime=0
        HOTtime=0
        try:
            COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
        except:
449
            print("download_hot_cold_reference_by_times: cold image for requested exposure time not found")
450
451
452
        try:
            HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
        except:
453
            print("download_hot_cold_reference_by_times: hot image for requested exposure time not found")
454
455
        images=[]
        if HOTtime!=0:
456
            HOT=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
457
458
459
            if HOT[0]:
                images.append(HOT[2])
        if COLDtime!=0:
460
            COLD=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
461
462
463
464
465
466
            if COLD[0]:
                images.append(COLD[2])
                
        if HOT[0] and COLD[0] and len(images)==2:
            return True,images,['hot','cold']
        else:
467
            print("download_hot_cold_reference_by_times: Error! hot and cold image for requested exposure time not found")
468
469
470
            return False,0,0
    else:
        return False,0,0
Holger Niemann's avatar
Holger Niemann committed
471
    
472
def download_background_by_program(port,program,exposure,version=0):
473
474
475
    '''
    returned exist,time,frame
    '''
476
    prog=get_program_from_PID(program)
477
478
479
480
481
482
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
483
        return download_background_by_times(port,starttime,stoptime,exposure,version)
484
    else:#except:
485
        print("download_background_by_program: Error! cannot find the program {0}".format(program))
Holger Niemann's avatar
Holger Niemann committed
486
487
        return False,0,0
    
488
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
489
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
490
    stream=portpathdict[OP]["AEF"+str(port)]+"background_"
491
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
492
493
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
494
495
496
497
498
499
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
500
        print('download_background_by_times: Error! ',e)
Holger Niemann's avatar
Holger Niemann committed
501
        goon=False
502
503
504
    except Exception as e:
        print('download_background_by_times: Error! ',e)
        goon=False
Holger Niemann's avatar
Holger Niemann committed
505
506
507
508
509
    if goon: 
        n=0
        backtimes=[0]
        for backpar in signal_list['values']:
            if backpar['structure']['Texp']==exposure:
510
                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
511
512
513
514
515
516
                    if backpar['structure']['filter']==camera_filter:
                        backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
                else:
                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
            n+=1
        if backtimes[0]!=0:
517
            backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
518
519
520
            if backdat[0]:                                    
                return backdat#[True,backdat[2]]
            else:
521
                print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
Holger Niemann's avatar
Holger Niemann committed
522
523
                return False,0,0
        else:
524
            print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
Holger Niemann's avatar
Holger Niemann committed
525
526
527
528
            return False,0,0
    else:
        return False,0,0

529
530
531
532
533
def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False,verbose=0):
    """
    OP1.2b function
    """
    exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version,verbose=verbose-1)
534
535
536
    camera=portcamdict["OP1.2b"]["AEF"+str(port)]  
    if verbose>50:
        print("camera is",camera)
537
538
539
540
541
542
543
544
545
    background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
    for frame in backgroundframes:
        background+=frame
    background=np.asarray(background/len(backgroundframes),dtype=np.uint16)
    if plot_it:
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image unnuced")
    if not camera.split("_")[0]=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
546
        background=apply_NUC([background],gain,offset)[0]
547
548
549
550
551
552
    if plot_it:        
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image nuced")
    return exist,btime[0],background

553
554
555
def download_raw_images_by_program(port,program,time_window=0,version=0,verbose=0):
    """
    """
556
557
558
559
560
561
562
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
563
564
565
566
567
        if type(time_window)==list or type(time_window)==np.ndarray:
            if len(time_window)>2 or time_window[0]>1000:
                raise Exception("download_raw_images_by_program: invalid input for the time_window!")
            tstart=time_window[0]
            tstop=time_window[1]
Holger Niemann's avatar
Holger Niemann committed
568
            if tstop<tstart:
569
                raise Exception("download_raw_images_by_program: endtime before starttime")
570
            return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
571
        else:
572
            if time_window==0:
573
                return download_raw_images_by_times(port,starttime,stoptime,version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
574
            else:
575
                return download_raw_images_by_times(port,starttime,int(starttime+time_window*1e9),version,verbose=verbose-1)
576
    except:
577
        print("download_raw_images_by_program: Error! cannot find the program")
Holger Niemann's avatar
Holger Niemann committed
578
579
        return False,0,0

580
def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
581
582
    """
    """
583
584
    OP=IR_tools.get_OP_by_time(time_ns=starttime)    
    if version==0:
585
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime)
586
587
588
589
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"  
    return download_images_by_times(larchivepath,starttime=starttime,stoptime=stoptime,version=version,intervalSize=intervalSize,verbose=verbose-1)
    
def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
590
591
    """
    """
592
593
594
595
    if '_raw' in larchivepath:
        typo=np.uint16          
    else:
        typo=np.float32
596
597
    if (stoptime-starttime)/intervalSize>1:        
        nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
598
        if verbose>0:
599
            print("download_images_by_times: time_window to large, splitting {0}s interval into {1} smaller fractions".format((stoptime-starttime)/1e9,nrinterv))
Holger Niemann's avatar
Holger Niemann committed
600
        intervalls=[]
601
        for i in range(nrinterv):
602
            intervalls.append(int(starttime-10+i*intervalSize))
Holger Niemann's avatar
Holger Niemann committed
603
        intervalls.append(stoptime)
604
605
606
        time=-1
        allimages=0
        success=False
607
        for i in range(nrinterv):
Holger Niemann's avatar
Holger Niemann committed
608
            try:
Holger Niemann's avatar
Holger Niemann committed
609
                res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1]))
Holger Niemann's avatar
Holger Niemann committed
610
                signal_list = json.loads(res.read().decode('utf-8'))
611
612
                res.close()                
                images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
613
#                signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
614
615
616
                t=np.array(signal_list['dimensions'])
                del signal_list
                if i==0:
617
                    allimages=images.copy()
Holger Niemann's avatar
Holger Niemann committed
618
619
                    time=t
                else:
620
                    allimages=allimages+images#np.append(ressignal,signal,axis=0)
Holger Niemann's avatar
Holger Niemann committed
621
                    time=np.append(time,t)
622
                del images,t
623
                success=True
Holger Niemann's avatar
Holger Niemann committed
624
            except urllib.error.URLError as e:
625
                print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e))                
626
627
            except Exception as e:
                print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e))   
628
        return success,time,allimages
Holger Niemann's avatar
Holger Niemann committed
629
630
    else:
        try:
631
632
            if verbose>100:
                print(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))
633
            res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))            
Holger Niemann's avatar
Holger Niemann committed
634
635
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
636
            images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
637
#            signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
638
639
            t=np.array(signal_list['dimensions'])
            del signal_list
640
            return True, t, images
Holger Niemann's avatar
Holger Niemann committed
641
        except urllib.error.URLError as e:
642
            print('download_images_by_times: Error! ',e)
Holger Niemann's avatar
Holger Niemann committed
643
            return False, 0,-1
644
645
646
        except Exception as e:
            print('download_images_by_times: Error! ',e)
            return False, 0,-1
647

648
if fastDL:
649
650
651
    def download_raw_images_by_program_via_png(port,program,time_window=0,version=0,threads=1,verbose=0):
        """
        """
652
        prog=get_program_from_PID(program)
653
654
655
#        try:
#            t_program = AKF_2.get_program_from_to(program)
#            prog =AKF_2.get_program_list(t_program[0], t_program[1])
656
        if not prog[0]:
657
658
            print("download_raw_images_by_program_via_png: Error! program not found, cannot download the data")
            return False,[0],[0]
659
        else:
660
661
            starttime=prog[1]['trigger']['1'][0]
            stoptime=prog[1]['trigger']['6'][0]
662
            OP=IR_tools.get_OP_by_time(time_ns=starttime)
663
            if version==0:
664
                version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime)
665
            larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
666
667
668
            return download_images_by_time_via_png(larchivepath,starttime,stoptime,
                                                   time_window=time_window,threads=threads,
                                                   version=version,verbose=verbose-1)
669
            
670
    def download_images_by_time_via_png(larchivepath,starttime,stoptime,time_window=0,
671
                                        threads=1,version=0,verbose=0,framerate='max'):
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
        """
        """
        success=True            
        if type(time_window)==list:
            tstart=time_window[0]
            tstop=time_window[1]
            if tstop<tstart:
                raise Exception("download_images_by_time_via_png: endtime before starttime")
            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
            stdate=stdate.isoformat()
            enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
            enddate=enddate.isoformat()
        else:
            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
            stdate=stdate.isoformat()
            if time_window==0:
                enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
689
690
                enddate=enddate.isoformat()
            else:
691
692
693
694
695
696
                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window)  
                enddate=enddate.isoformat()
            #"2017-11-15 08:00:00"
        try:
            times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
        except Exception as E:
697
            print('download_images_by_time_via_png: Error loading times from ',larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))
698
            raise Warning(E)
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
            return False,[0],[0]  
            
        if framerate!='max': 
            if verbose>5:
                print("download_images_by_time_via_png: downsampling active; number of original frames:",len(times))
            if type(framerate)==float or type(framerate)==int:
                realrate=1e9/((times[0]-times[-1])[0]/len(times))
                if framerate<realrate:
                    steps=int(round(realrate/framerate,0))
                    dummy=[]
                    for i in range(0,len(times),steps):
                        dummy.append(times[i])
                    times=dummy
            if verbose>5:
                print("download_images_by_time_via_png: downsampling active; number of downsampeld frames:",len(times),steps)
714
715
716
717
718
719
720
721
722
723
        time=[]
        images=[]
        lnt=len(times)
        if threads==1:
            for i in range(lnt):
                ele=times[lnt-1-i]
                imag=download_last_raw_image_by_time(larchivepath,ele[0]-10,ele[0]+10)
                if imag[0]:
                    time.append(ele[0])
                    images.append(np.array(imag[1],dtype=np.uint16))
Holger Niemann's avatar
Holger Niemann committed
724
                else:
725
726
727
728
729
730
731
732
733
734
735
736
737
738
                    success=False
            return success,np.array(time),images
        else:
            tim=[]
            for i in range(lnt):
                tim.append(times[lnt-1-i][0])
            intervalls=[]
            intervalSize=int(lnt/threads)
            for i in range(threads):
                intervalls.append(int(i*intervalSize))
            intervalls.append(lnt)
            jobs = []
            resultdict = []
            for i in range(threads):
739
                if verbose>0:
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
                    print("download_images_by_time_via_png: Start Thread ",i+1)                
                TH=download_images_thread(i,larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,version)
                jobs.append(TH)
                TH.start()
            for p in jobs:
                p.join()
            if verbose>0:
                print("download_images_by_time_via_png: all threads are done")
            order=[]
            for ele in resultdict:
                order.append(ele[0])
                if len(np.where(np.asarray(ele[1])==False)[0])>0:
                    success=False                
            times=[]#np.array(resultdict[order.index(0)][2])
            images=[]#np.array(resultdict[order.index(0)][3])
            for i in range(threads):
                images=images+resultdict[order.index(i)][3]
                times=times+resultdict[order.index(i)][2]
                resultdict[order.index(i)]=[]
            del resultdict
            return success,np.array(times),images
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
      
    class download_images_thread(threading.Thread):
        def __init__(self, threadID, larchivepath,times,resultlist,version=0):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            self.resultlist.append([self.threadID,successes,time,images])
783
    
784
    class convert_raw_to_temp_thread(threading.Thread):
785
786
787
        def __init__(self,larchivepath,times,resultlist,threadID,version=0,background=0,
                     LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,
                     give_ERROR=False,FOV=0,badpixels=[],verbose=0):
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
804
            self.verbose=verbose
805
806
807
808
809
810
811
812
813
814
815
816
817
        def run(self):
            images=[]
            time=[]
            successes=[]        
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            if self.give_ERROR:
818
                sucess,images,error_images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR,verbose=self.verbose-1)
819
            else:
820
                success,images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR,verbose=self.verbose-1)
821
822
823
            
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
824
825
826
            if self.verbose>0:
                print(datetime.datetime.now(),"convert_raw_to_temp_thread: correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
827
            self.resultlist.append([self.threadID,successes,time,images])
828
    
829
    class NUC_raw_thread(threading.Thread):
830
831
        def __init__ (self,larchivepath,times,resultlist,threadID,version=0,background=0,
                      LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,
832
                      give_ERROR=False,FOV=0,badpixels=[],verbose=0):
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
849
            self.verbose=verbose
850
851
852
853
854
855
856
857
858
859
860
861
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
862
863
            if self.verbose>0:
                print(datetime.datetime.now(),"NUC_raw_thread: applying NUC")
864
865
            if self.give_ERROR:
                images,error_images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
866
            else:
867
868
869
                images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
Peter Drewelow's avatar
Peter Drewelow committed
870
                
871
872
873
            if self.verbose>0:
                print(datetime.datetime.now(),"NUC_raw_thread: correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
874
            self.resultlist.append([self.threadID,successes,time,images]) 
875

876
def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,verbose=0,Test=True):
877
878
    """
    """
879
880
881
882
    if Test:
        larchivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+"W7X/QSR07_FLIR/AEK51_raw"
    else:
        larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
883
884
885
    if type(time_window)==list:
        tstart=time_window[0]
        tstop=time_window[1]
886
        if tstop<tstart:
887
            raise Exception("download_raw_FLIR_images_via_png: endtime before starttime")
888
889
890
891
892
893
        starttime=int(t1-100+tstart*1e9)
        endtime=int(t1+tstop*1e9)
#        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
#        stdate=stdate.isoformat()
#        enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)  
#        enddate=enddate.isoformat()
894
    else:
895
        starttime=int(t1-100)
896
        if time_window==0:
897
            endtime=t6
898
        else:
899
900
901
902
903
904
905
906
907
            endtime=int((t1)+time_window*1e9)
#        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
#        stdate=stdate.isoformat()
#        if time_window==0:
#            enddate=datetime.datetime.utcfromtimestamp(t6/1e9)        
#            enddate=enddate.isoformat()
#        else:
#            enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_window)  
#            enddate=enddate.isoformat()
908
        #"2017-11-15 08:00:00"
909
    typ1_version=get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM",Test=Test,t_from=tstart)
910
911
912
913
914
915
    if typ1_version==None:
        mode=1
    else:
        mode=2
    if mode==1:
        ### okay the data is stored probably as different sets (so recorded with FLIR software)
916
        timess,versions=get_FLIR_Pset_times(starttime,endtime,version,Test=Test)#stdate,enddate,version,Test=Test)
917
        if verbose>0:
918
            print("download_raw_FLIR_images_via_png: found {0} frames in the different sets".format([np.shape(x) for x in timess]))
919
920
921
        ### we not know the times for all presets for the program, now we have to download them, if times is not empty
        exist,time,frames,tsets=download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=verbose-1)            
    elif mode==2:##the data is stored only in one datastream, at least this datastream exist
922
        raise Exception("download_raw_FLIR_images_via_png: not implemented")
923
924
        #download data and downlaod LUT
        
925
926
927
928
929
930
931
932
933
934
935
936
937
#        time=[]
#        images=[]
#        lnt=len(times)
#        if threads==1:
#            for i in range(lnt):
#                ele=times[lnt-1-i]
#                imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
#                if imag[0]:
#                    time.append(ele[0])
#                    images.append(np.array(imag[1],dtype=np.uint16))
#                else:
#                    success=False
#            return success,np.array(time),images
938
    else:
939
        raise Exception("download_raw_FLIR_images_via_png: cannot identifie the way the searched data is stored!")                        
940
941
942

    return exist,time,frames,mode,tsets,versions

943
def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0,Test=True):
944
945
    """
    """
946
947
948