downloadversionIRdata.py 151 KB
Newer Older
Holger Niemann's avatar
Holger Niemann committed
1
2
3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
4
updated on Tue Aug 21 10:20:00 2018
5
last update on Fr Nov 23 15:37:00 2018
Holger Niemann's avatar
Holger Niemann committed
6

7
Version: 3.2.1
8
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
Holger Niemann's avatar
Holger Niemann committed
9
10
11
12
@author: holn
"""

import numpy as np
13
import IR_image_tools as IR_tools
14
15
16
17
18
from IR_config_constants import archivepath,portcamdict,camlimdict, \
                                valid_FOV_circle,valid_background_rectangle, \
                                TC_port,TC_channel,IRCamColdframes_fittingpath, \
                                exJet,portpathdict,project,project_ana, \
                                heatflux_requestlist_path#,exJet_trans
19
20
21
22
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
Holger Niemann's avatar
Holger Niemann committed
23
24
25
import datetime
import urllib
import json
26
27
from PIL import Image
from io import BytesIO
28
from os.path import join
29
import matplotlib.pyplot as plt
30
31
try:          
    import threading
32
    import archivedb as AKF_2
33
34
35
36
    fastDL=True
except Exception as E:
    print(E)
    fastDL=False
Holger Niemann's avatar
Holger Niemann committed
37

38

39
try:
40
    config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
41
42
43
44
45
    FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0]   
    del FOV
except:
    config_path=""

46

47
def get_latest_version(stream,project="W7X",Test=False,t_from=None,t_to=None,program=None):
48
49
50
51
52
     """Find out latest version of given stream
     :param stream   url stream of interest
     :return int of version number of None if non-versioned or non-existing stream
     author: G. Schlisio, edit by holn
     """
53
54
55
56
     if Test:
         base="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
     else:
         base=archivepath    
57
58
59
60
61
62
63
64

    
     if program==None and t_from==None and t_to==None:         
         request =urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})            
     else:
         if program is not None:
             prog=get_program_from_PID(program)
             if prog[0]:
65
                 t_from=prog[1]['trigger']['0'][0]
66
67
68
69
70
71
                 t_to=prog[1]['trigger']['6'][0]
         elif t_from is not None and t_to==None:
             t_to=int(t_from+100e9)
         elif t_from==None and t_to is not None:
             t_from=int(t_to-10e9)
         request =urllib.request.Request(base +project+"/" + stream + "/_versions.json?from="+str(t_from)+"&upto="+str(t_to), headers={"Accept": "application/json"})
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
     try:
         response = urllib.request.urlopen(request)
         d = json.loads(response.read().decode('utf-8'))
     except urllib.error.HTTPError as ex:
         msg = ex.read()
         raise RuntimeError(msg)
     # detect unversioned or non-existing stream
     if d["versionInfo"] == []:
         return None
     versions = []
     for i in d['versionInfo']:
         versions.append(i['number'])

     return max(versions)

Holger Niemann's avatar
Holger Niemann committed
87
88
89
90
91
92
93
94
95
96
97
def TimeToNs(date,time):
    """
    TimeToNs(date,time)
    (list)  date    [year,month,day]
    (list)  time    [hours,minutes,seconds,microseconds]
    """ 
    date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3])
    div=date_time-datetime.datetime(1970,1,1,0,0,0)
    nsdate=div.total_seconds()*1e9
    return nsdate

98
def read_program(timestamp_start,timestamp_end=0,tol=60):
Holger Niemann's avatar
Holger Niemann committed
99
100
101
102
    """
    read_program()
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
103
    if timestamp_end==0:
104
105
106
107
        timestamp_end = timestamp_start
    jsonurl = '{0}={1}&upto={2}'.format(program_url,
                                        int(timestamp_start-tol*1e9),
                                        int(timestamp_end+tol*1e9))
Holger Niemann's avatar
Holger Niemann committed
108
109
110
111
112
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw=res.read()
        res.close()
    except urllib.error.URLError as e:
113
        print('read_program: Error opening URL')
Holger Niemann's avatar
Holger Niemann committed
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
        print(e)
        return False,0
    else:
        prog_string=prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl=prog_list['programs'][0]
        return True, pl

def read_restdb_old(request_url):
    """
    Reads JSON data from W7-X REST API
    Returns: 
        valid: access ok
        t: numpy-array of time
        signal: numpy-array of requested signals
129
    by H. Thomsen
Holger Niemann's avatar
Holger Niemann committed
130
131
132
133
    """
    try:
        res = urllib.request.urlopen(request_url)
    except urllib.error.URLError as e:
134
        print('read_restdb_old: Error ', e)
Holger Niemann's avatar
Holger Niemann committed
135
136
137
138
139
140
141
142
143
144
        return False, 0, -1
    else:
        signal_raw=res.read()
        res.close()
        signal_string=signal_raw.decode(encoding='UTF-8')
        signal_list = json.loads(signal_string)
        signal0=np.array(signal_list['values'])
        t=np.array(signal_list['dimensions'])
        return True, t, signal0

145
def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
Holger Niemann's avatar
Holger Niemann committed
146
147
148
    """
    download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
        time in ns
149
    Have to swap 11, 21 until correction in the database
Holger Niemann's avatar
Holger Niemann committed
150
    """    
151
    OP=IR_tools.get_OP_by_time(time_ns=time)
152
153
154
155
    if port == 21:
        port = 11
    elif port == 11:
        port = 21
156
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
157
    if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
158
        query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
159
    elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
Holger Niemann's avatar
Holger Niemann committed
160
161
        query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
162
        print("download_LUT: Error! Camera unknown, stopping here.")
Holger Niemann's avatar
Holger Niemann committed
163
        raise Exception
164
    if version==0:
165
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM")
166
    if verbose>0:
167
        print("download_LUT: LUT V"+str(version)+" is used")
Holger Niemann's avatar
Holger Niemann committed
168
    #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
169
    LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
170
171
    if LUTpar[0]:
        LUTid=LUTpar[2][0]['structure'][query]
172
        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
173
        if LUTs[0]:
174
            LUTs=LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
Holger Niemann's avatar
Holger Niemann committed
175
176
177
178
            LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True,LUT
        else:
179
            print("download_LUT: Warning! unable to download the LUTs")
Holger Niemann's avatar
Holger Niemann committed
180
181
182
183
            del LUTpar, LUTs
            return False,0
    else:
        del LUTpar
184
        print("download_LUT: Warning! unable to find LUTs, check your request")
Holger Niemann's avatar
Holger Niemann committed
185
186
        return False,0

187
188
189
190
191
192
193
194

def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
    
    Read LUT V3 from local files for testing new calibration.
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
195
    OP = IR_tools.get_OP_by_time(time_ns=this_time)
196
197
198
199
200
201
202
203
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
        with open(LUT_dir+"\\"+filename) as data_file:  
            jfile=json.load(data_file)
        LUT=np.array(jfile['LUT']).swapaxes(0,1)
        if verbose>0:
204
            print("read_LUT_from_file: succesfully loaded V3 LUT from local directory")
205
206
        return True, LUT
    except Exception as E:
207
        print("read_LUT_from_file: ERROR in loading V3 LUTs",E)
208
209
        return False, []
    
210
def download_NUC_by_program(port,program,exposure,version=0):
211
    prog=get_program_from_PID(program)
212
213
214
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
Holger Niemann's avatar
Holger Niemann committed
215
216
217
218
219
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_NUC_by_times(port,starttime,stoptime,exposure,version)
    else:
220
#    except:
221
        print("download_NUC_by_program: cannot find the program")
Holger Niemann's avatar
Holger Niemann committed
222
223
        return False,0,0

224
def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
225
    project="W7X"
226
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
227
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
Holger Niemann's avatar
Holger Niemann committed
228
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
229
    if version==0:
230
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
231
232
233
234
235
236
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
237
        print('download_NUC_by_times: Error! ', e)
Holger Niemann's avatar
Holger Niemann committed
238
239
240
241
242
243
244
245
246
247
248
        goon=False
    if goon:        
        n=0
        nuctimes=[0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp']==exposure:
                nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i=NUCpar['structure']['offset_index']                
            n+=1 
        if nuctimes[0]!=0:
249
            NUC=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
250
251
252
253
            if NUC[0]:
                images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)                    
                return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
            else:
254
                print("download_NUC_by_times: NUC image for requested exposure time not found")
Holger Niemann's avatar
Holger Niemann committed
255
256
                return False, 0, 0
        else:
257
            print("download_NUC_by_times: NUC image for requested exposure time not found")
Holger Niemann's avatar
Holger Niemann committed
258
259
260
            return False,0,0
    else:
        return False,0,0
261

262
def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
263
    prog=get_program_from_PID(program)
264
265
266
267
268
269
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
270
        return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose-1)
271
272
    else:
#    except:
273
        print("get_NUC_by_program: Error! cannot find the program")
274
275
        return False,0,0
    
276
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
277
278
279
280
281
    """
    Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera
    in a given port and time interval. Depending on time and camera, reconstruct
    teh cold frame and/or compute the gain,offset from the hot and cold frame.
    """
282
283
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if OP=="OP1.2a":
284
285
286
287
        t1=endtime
        t0=starttime
        prog=read_program(t1)
        program=prog[1]['id']
288
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):
289
290
            if verbose>0:
                print("get_NUC_by_times: rebuilding coldframe")
291
292
293
294
295
296
297
298
            #use any way the rebuilt coldframe.
            sT = get_sensor_temp_by_program(port, program)[2][0]
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
299
                raise Exception("get_NUC_by_times: unable to download reference frames")
300
301
302
303
            filestring = 'AEF' + str(port) + '_et' + str(t_exp)
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
Peter Drewelow's avatar
Peter Drewelow committed
304
305
            if verbose>0:
                print('get_NUC_by_times: calculate gain, offset and bad pixels')
306
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
307
308
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
309
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)
310
311
312
313
            gain_error=0
            offset_error=0        
            return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        elif (port == 11) or (port == 21):
Peter Drewelow's avatar
Peter Drewelow committed
314
315
            if verbose>0:
                print('get_NUC_by_times: downloading NUC')
316
317
318
319
320
321
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
322
                raise Exception("get_NUC_by_times: unable to download reference frames")
323
324
            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
            if NUC_DL[0]==False:
325
                raise Exception("get_NUC_by_times: NUC was not found")
326
327
            else:
                ci = NUC_DL[1][2]
328
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1)
329
330
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
331
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)                                
332
333
            gain_error=0
            offset_error=0
334
            return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
335
        else:
Peter Drewelow's avatar
Peter Drewelow committed
336
337
            if verbose>0:
                print('get_NUC_by_times: downloading NUC')
338
339
340
341
342
343
344
345
346
            return download_NUC_by_times(port,starttime,endtime,t_exp)
    elif OP=="OP1.2b":
        gain_error=0
        offset_error=0
        gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
        if gotit:
            coldref=hotcold[1]
            hotref=hotcold[0]
        else:
347
            raise Exception("get_NUC_by_times: unable to download reference frames")
348
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
349
        exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0,verbose=verbose-1)
350
351
352
353
354
355
356
        if exist:
            cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
            for ele in coldframes:
                cold+=ele                
            cold=np.asarray(cold/len(coldframes),dtype=np.uint16)        
        else:
            return False,[0],[0]
357
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
358
        badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)
359
360
        return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
    else:
361
        raise Exception("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP")
362

363
def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0,verbose=0):
364
365
366
367
    """
    download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
    """
Holger Niemann's avatar
Holger Niemann committed
368
    gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(endtime-20e6))
369
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
370
371
372
    if gotit:
        expinds=np.where(texp_t==t_exp)[0]
        if len(expinds)==0:
373
            print("download_calibration_raw_files_by_time: Error! cannot find the exposure time in the given data")
374
375
            return False,[0],[0]                               
    else:
376
        print("download_calibration_raw_files_by_time: Error! exposure time not found")
377
        return False,[0],[0]
Holger Niemann's avatar
Holger Niemann committed
378
    gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(endtime-20e6))
379
    if gotitf:
Holger Niemann's avatar
Holger Niemann committed
380
        typiinds=np.where(values_f[expinds]==frametype)[0]
381
        if len(typiinds)>0:
Holger Niemann's avatar
Holger Niemann committed
382
383
384
            ref_t=[np.min(timef[expinds][typiinds]),np.max(timef[expinds][typiinds])]
#            print((ref_t[1]-ref_t[0])/1e9)
#            print(len(timef),len(timef[expinds]),len(timef[expinds][typiinds]))
385
        else:#okay the early data stuff or strange stuff
386
387
            if verbose>0:
                print("download_calibration_raw_files_by_time: frame type was not identified assuming that the first part is the cold and the second one the background")
388
389
390
391
392
            frametimes=time_t[expinds]
            diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
            turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
            if frametype==0:
                ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
Holger Niemann's avatar
Holger Niemann committed
393
                print((ref_t[1]-ref_t[0])/1e9)
394
395
396
            elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
                ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
Holger Niemann's avatar
Holger Niemann committed
397
                print((ref_t[1]-ref_t[0])/1e9)
398
            else:
399
                raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
400
    else:
401
        print("download_calibration_raw_files_by_time: Error! frametype not found")
402
403
404
405
406
407
408
409
410
411
412
413
414
        frametimes=time_t[expinds]
        diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
        turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
        if frametype==0:
            ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
            print((ref_t[1]-ref_t[0])/1e9)
        elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
            ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
            print((ref_t[1]-ref_t[0])/1e9)
        else:
            raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
#        return False,[0],[0]
415
416
    t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
    t1date=t1date.isoformat()
Holger Niemann's avatar
Holger Niemann committed
417
    t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
418
419
    t0date=t0date.isoformat()
    if version==0:
420
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=endtime)
421
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
422
    timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
Holger Niemann's avatar
Holger Niemann committed
423
    t_offset=timest0[-1][0]-time_t[0]
424
    if t_offset>10e6:
425
426
        if verbose>0:
            print("download_calibration_raw_files_by_time: time offset detected, try to correct this, offset is",t_offset)
427
428
429
    else:
        t_offset=0
#            print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
430
    return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset,verbose=verbose-1)
431
432
        
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
433
434
435
436
    """
    Loads the most recent hot and cold calibration frames for a starttime.
    Uses first calibration frames if time is not defined.
    """
437
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
438
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
439
440
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
441
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime)
442
443
444
445
446
447
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
448
        print('download_hot_cold_reference_by_times: Error! ',e)
449
450
451
452
453
454
455
        goon=False
    if goon:        
        COLDtime=0
        HOTtime=0
        try:
            COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
        except:
456
            print("download_hot_cold_reference_by_times: cold image for requested exposure time not found")
457
458
459
        try:
            HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
        except:
460
            print("download_hot_cold_reference_by_times: hot image for requested exposure time not found")
461
462
        images=[]
        if HOTtime!=0:
463
            HOT=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
464
465
466
            if HOT[0]:
                images.append(HOT[2])
        if COLDtime!=0:
467
            COLD=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
468
469
470
471
472
473
            if COLD[0]:
                images.append(COLD[2])
                
        if HOT[0] and COLD[0] and len(images)==2:
            return True,images,['hot','cold']
        else:
474
            print("download_hot_cold_reference_by_times: Error! hot and cold image for requested exposure time not found")
475
476
477
            return False,0,0
    else:
        return False,0,0
Holger Niemann's avatar
Holger Niemann committed
478
    
479
def download_background_by_program(port,program,exposure,version=0):
480
481
482
    '''
    returned exist,time,frame
    '''
483
    prog=get_program_from_PID(program)
484
485
486
487
488
489
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
490
        return download_background_by_times(port,starttime,stoptime,exposure,version)
491
    else:#except:
492
        print("download_background_by_program: Error! cannot find the program {0}".format(program))
Holger Niemann's avatar
Holger Niemann committed
493
494
        return False,0,0
    
495
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
496
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
497
    stream=portpathdict[OP]["AEF"+str(port)]+"background_"
498
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
499
500
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
501
502
503
504
505
506
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
507
        print('download_background_by_times: Error! ',e)
Holger Niemann's avatar
Holger Niemann committed
508
509
510
511
512
513
        goon=False
    if goon: 
        n=0
        backtimes=[0]
        for backpar in signal_list['values']:
            if backpar['structure']['Texp']==exposure:
514
                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
515
516
517
518
519
520
                    if backpar['structure']['filter']==camera_filter:
                        backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
                else:
                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
            n+=1
        if backtimes[0]!=0:
521
            backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
522
523
524
            if backdat[0]:                                    
                return backdat#[True,backdat[2]]
            else:
525
                print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
Holger Niemann's avatar
Holger Niemann committed
526
527
                return False,0,0
        else:
528
            print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
Holger Niemann's avatar
Holger Niemann committed
529
530
531
532
            return False,0,0
    else:
        return False,0,0

533
534
535
536
537
def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False,verbose=0):
    """
    OP1.2b function
    """
    exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version,verbose=verbose-1)
538
539
540
    camera=portcamdict["OP1.2b"]["AEF"+str(port)]  
    if verbose>50:
        print("camera is",camera)
541
542
543
544
545
546
547
548
549
    background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
    for frame in backgroundframes:
        background+=frame
    background=np.asarray(background/len(backgroundframes),dtype=np.uint16)
    if plot_it:
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image unnuced")
    if not camera.split("_")[0]=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
550
        background=apply_NUC([background],gain,offset)[0]
551
552
553
554
555
556
    if plot_it:        
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image nuced")
    return exist,btime[0],background

557
558
559
def download_raw_images_by_program(port,program,time_window=0,version=0,verbose=0):
    """
    """
560
561
562
563
564
565
566
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
567
568
569
570
571
        if type(time_window)==list or type(time_window)==np.ndarray:
            if len(time_window)>2 or time_window[0]>1000:
                raise Exception("download_raw_images_by_program: invalid input for the time_window!")
            tstart=time_window[0]
            tstop=time_window[1]
Holger Niemann's avatar
Holger Niemann committed
572
            if tstop<tstart:
573
                raise Exception("download_raw_images_by_program: endtime before starttime")
574
            return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
575
        else:
576
            if time_window==0:
577
                return download_raw_images_by_times(port,starttime,stoptime,version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
578
            else:
579
                return download_raw_images_by_times(port,starttime,int(starttime+time_window*1e9),version,verbose=verbose-1)
580
    except:
581
        print("download_raw_images_by_program: Error! cannot find the program")
Holger Niemann's avatar
Holger Niemann committed
582
583
        return False,0,0

584
def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
585
586
    """
    """
587
588
    OP=IR_tools.get_OP_by_time(time_ns=starttime)    
    if version==0:
589
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime)
590
591
592
593
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"  
    return download_images_by_times(larchivepath,starttime=starttime,stoptime=stoptime,version=version,intervalSize=intervalSize,verbose=verbose-1)
    
def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
594
595
    """
    """
596
597
598
599
    if '_raw' in larchivepath:
        typo=np.uint16          
    else:
        typo=np.float32
600
601
    if (stoptime-starttime)/intervalSize>1:        
        nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
602
        if verbose>0:
603
            print("download_images_by_times: time_window to large, splitting {0}s interval into {1} smaller fractions".format((stoptime-starttime)/1e9,nrinterv))
Holger Niemann's avatar
Holger Niemann committed
604
        intervalls=[]
605
        for i in range(nrinterv):
606
            intervalls.append(int(starttime-10+i*intervalSize))
Holger Niemann's avatar
Holger Niemann committed
607
        intervalls.append(stoptime)
608
609
610
        time=-1
        allimages=0
        success=False
611
        for i in range(nrinterv):
Holger Niemann's avatar
Holger Niemann committed
612
            try:
Holger Niemann's avatar
Holger Niemann committed
613
                res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1]))
Holger Niemann's avatar
Holger Niemann committed
614
                signal_list = json.loads(res.read().decode('utf-8'))
615
616
                res.close()                
                images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
617
#                signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
618
619
620
                t=np.array(signal_list['dimensions'])
                del signal_list
                if i==0:
621
                    allimages=images.copy()
Holger Niemann's avatar
Holger Niemann committed
622
623
                    time=t
                else:
624
                    allimages=allimages+images#np.append(ressignal,signal,axis=0)
Holger Niemann's avatar
Holger Niemann committed
625
                    time=np.append(time,t)
626
                del images,t
627
                success=True
Holger Niemann's avatar
Holger Niemann committed
628
            except urllib.error.URLError as e:
629
                print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e))                
630
        return success,time,allimages
Holger Niemann's avatar
Holger Niemann committed
631
632
    else:
        try:
633
634
            if verbose>100:
                print(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))
635
            res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))            
Holger Niemann's avatar
Holger Niemann committed
636
637
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
638
            images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
639
#            signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
640
641
            t=np.array(signal_list['dimensions'])
            del signal_list
642
            return True, t, images
Holger Niemann's avatar
Holger Niemann committed
643
        except urllib.error.URLError as e:
644
            print('download_images_by_times: Error! ',e)
Holger Niemann's avatar
Holger Niemann committed
645
            return False, 0,-1
646

647
if fastDL:
648
649
650
    def download_raw_images_by_program_via_png(port,program,time_window=0,version=0,threads=1,verbose=0):
        """
        """
651
        prog=get_program_from_PID(program)
652
653
654
#        try:
#            t_program = AKF_2.get_program_from_to(program)
#            prog =AKF_2.get_program_list(t_program[0], t_program[1])
655
        if not prog[0]:
656
657
            print("download_raw_images_by_program_via_png: Error! program not found, cannot download the data")
            return False,[0],[0]
658
        else:
659
660
            starttime=prog[1]['trigger']['1'][0]
            stoptime=prog[1]['trigger']['6'][0]
661
            OP=IR_tools.get_OP_by_time(time_ns=starttime)
662
            if version==0:
663
                version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime)
664
            larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
665
666
667
            return download_images_by_time_via_png(larchivepath,starttime,stoptime,
                                                   time_window=time_window,threads=threads,
                                                   version=version,verbose=verbose-1)
668
            
669
    def download_images_by_time_via_png(larchivepath,starttime,stoptime,time_window=0,
670
                                        threads=1,version=0,verbose=0,framerate='max'):
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
        """
        """
        success=True            
        if type(time_window)==list:
            tstart=time_window[0]
            tstop=time_window[1]
            if tstop<tstart:
                raise Exception("download_images_by_time_via_png: endtime before starttime")
            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
            stdate=stdate.isoformat()
            enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
            enddate=enddate.isoformat()
        else:
            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
            stdate=stdate.isoformat()
            if time_window==0:
                enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
688
689
                enddate=enddate.isoformat()
            else:
690
691
692
693
694
695
                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window)  
                enddate=enddate.isoformat()
            #"2017-11-15 08:00:00"
        try:
            times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
        except Exception as E:
696
            print('download_images_by_time_via_png: Error loading times from ',larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))
697
            raise Warning(E)
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
            return False,[0],[0]  
            
        if framerate!='max': 
            if verbose>5:
                print("download_images_by_time_via_png: downsampling active; number of original frames:",len(times))
            if type(framerate)==float or type(framerate)==int:
                realrate=1e9/((times[0]-times[-1])[0]/len(times))
                if framerate<realrate:
                    steps=int(round(realrate/framerate,0))
                    dummy=[]
                    for i in range(0,len(times),steps):
                        dummy.append(times[i])
                    times=dummy
            if verbose>5:
                print("download_images_by_time_via_png: downsampling active; number of downsampeld frames:",len(times),steps)
713
714
715
716
717
718
719
720
721
722
        time=[]
        images=[]
        lnt=len(times)
        if threads==1:
            for i in range(lnt):
                ele=times[lnt-1-i]
                imag=download_last_raw_image_by_time(larchivepath,ele[0]-10,ele[0]+10)
                if imag[0]:
                    time.append(ele[0])
                    images.append(np.array(imag[1],dtype=np.uint16))
Holger Niemann's avatar
Holger Niemann committed
723
                else:
724
725
726
727
728
729
730
731
732
733
734
735
736
737
                    success=False
            return success,np.array(time),images
        else:
            tim=[]
            for i in range(lnt):
                tim.append(times[lnt-1-i][0])
            intervalls=[]
            intervalSize=int(lnt/threads)
            for i in range(threads):
                intervalls.append(int(i*intervalSize))
            intervalls.append(lnt)
            jobs = []
            resultdict = []
            for i in range(threads):
738
                if verbose>0:
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
                    print("download_images_by_time_via_png: Start Thread ",i+1)                
                TH=download_images_thread(i,larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,version)
                jobs.append(TH)
                TH.start()
            for p in jobs:
                p.join()
            if verbose>0:
                print("download_images_by_time_via_png: all threads are done")
            order=[]
            for ele in resultdict:
                order.append(ele[0])
                if len(np.where(np.asarray(ele[1])==False)[0])>0:
                    success=False                
            times=[]#np.array(resultdict[order.index(0)][2])
            images=[]#np.array(resultdict[order.index(0)][3])
            for i in range(threads):
                images=images+resultdict[order.index(i)][3]
                times=times+resultdict[order.index(i)][2]
                resultdict[order.index(i)]=[]
            del resultdict
            return success,np.array(times),images
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
      
    class download_images_thread(threading.Thread):
        def __init__(self, threadID, larchivepath,times,resultlist,version=0):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            self.resultlist.append([self.threadID,successes,time,images])
782
    
783
    class convert_raw_to_temp_thread(threading.Thread):
784
785
786
        def __init__(self,larchivepath,times,resultlist,threadID,version=0,background=0,
                     LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,
                     give_ERROR=False,FOV=0,badpixels=[],verbose=0):
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
803
            self.verbose=verbose
804
805
806
807
808
809
810
811
812
813
814
815
816
        def run(self):
            images=[]
            time=[]
            successes=[]        
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            if self.give_ERROR:
817
                sucess,images,error_images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR,verbose=self.verbose-1)
818
            else:
819
                success,images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR,verbose=self.verbose-1)
820
821
822
            
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
823
824
825
            if self.verbose>0:
                print(datetime.datetime.now(),"convert_raw_to_temp_thread: correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
826
            self.resultlist.append([self.threadID,successes,time,images])
827
    
828
    class NUC_raw_thread(threading.Thread):
829
830
        def __init__ (self,larchivepath,times,resultlist,threadID,version=0,background=0,
                      LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,
831
                      give_ERROR=False,FOV=0,badpixels=[],verbose=0):
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
848
            self.verbose=verbose
849
850
851
852
853
854
855
856
857
858
859
860
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
861
862
            if self.verbose>0:
                print(datetime.datetime.now(),"NUC_raw_thread: applying NUC")
863
864
            if self.give_ERROR:
                images,error_images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
865
            else:
866
867
868
                images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
Peter Drewelow's avatar
Peter Drewelow committed
869
                
870
871
872
            if self.verbose>0:
                print(datetime.datetime.now(),"NUC_raw_thread: correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
873
            self.resultlist.append([self.threadID,successes,time,images]) 
874

875
def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,verbose=0,Test=True):
876
877
    """
    """
878
879
880
881
    if Test:
        larchivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+"W7X/QSR07_FLIR/AEK51_raw"
    else:
        larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
882
883
884
    if type(time_window)==list:
        tstart=time_window[0]
        tstop=time_window[1]
885
        if tstop<tstart:
886
            raise Exception("download_raw_FLIR_images_via_png: endtime before starttime")
887
888
889
890
891
892
        starttime=int(t1-100+tstart*1e9)
        endtime=int(t1+tstop*1e9)
#        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
#        stdate=stdate.isoformat()
#        enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)  
#        enddate=enddate.isoformat()
893
    else:
894
        starttime=int(t1-100)
895
        if time_window==0:
896
            endtime=t6
897
        else:
898
899
900
901
902
903
904
905
906
            endtime=int((t1)+time_window*1e9)
#        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
#        stdate=stdate.isoformat()
#        if time_window==0:
#            enddate=datetime.datetime.utcfromtimestamp(t6/1e9)        
#            enddate=enddate.isoformat()
#        else:
#            enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_window)  
#            enddate=enddate.isoformat()
907
        #"2017-11-15 08:00:00"
908
    typ1_version=get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM",Test=Test,t_from=tstart)
909
910
911
912
913
914
    if typ1_version==None:
        mode=1
    else:
        mode=2
    if mode==1:
        ### okay the data is stored probably as different sets (so recorded with FLIR software)
915
        timess,versions=get_FLIR_Pset_times(starttime,endtime,version,Test=Test)#stdate,enddate,version,Test=Test)
916
        if verbose>0:
917
            print("download_raw_FLIR_images_via_png: found {0} frames in the different sets".format([np.shape(x) for x in timess]))
918
919
920
        ### we not know the times for all presets for the program, now we have to download them, if times is not empty
        exist,time,frames,tsets=download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=verbose-1)            
    elif mode==2:##the data is stored only in one datastream, at least this datastream exist
921
        raise Exception("download_raw_FLIR_images_via_png: not implemented")
922
923
        #download data and downlaod LUT
        
924
925
926
927
928
929
930
931
932
933
934
935
936
#        time=[]
#        images=[]
#        lnt=len(times)
#        if threads==1:
#            for i in range(lnt):
#                ele=times[lnt-1-i]
#                imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
#                if imag[0]:
#                    time.append(ele[0])
#                    images.append(np.array(imag[1],dtype=np.uint16))
#                else:
#                    success=False
#            return success,np.array(time),images
937
    else:
938
        raise Exception("download_raw_FLIR_images_via_png: cannot identifie the way the searched data is stored!")                        
939
940
941

    return exist,time,frames,mode,tsets,versions

942
def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0,Test=True):
943
944
    """
    """
945
946
947
948
    if Test:
        larchivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+"W7X/QSR07_FLIR/AEK51_raw"
    else:
        larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
949
950
    if version==0:
        versions=[]
951
952
953
954
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS0_DATASTREAM",Test=Test,t_from=stdate))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS1_DATASTREAM",Test=Test,t_from=stdate))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS2_DATASTREAM",Test=Test,t_from=stdate))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS3_DATASTREAM",Test=Test,t_from=stdate))
955
956
957
    else:
        versions=[version,version,version,version]
    timess=[]
958
959
960
    if version==0:
        if verbose>0:
            print('get_FLIR_Pset_times: no version specified search for data in all versions')
961
962
963