downloadversionIRdata.py 123 KB
Newer Older
Holger Niemann's avatar
Holger Niemann committed
1
2
3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
4
updated on Tue Aug 21 10:20:00 2018
5
last update on Fr Nov 23 15:37:00 2018
Holger Niemann's avatar
Holger Niemann committed
6

7
Version: 3.0.2
8
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
Holger Niemann's avatar
Holger Niemann committed
9
10
11
12
@author: holn
"""

import numpy as np
13
import IR_image_tools as IR_tools
14
from IR_config_constants import archivepath,portcamdict,camlimdict,valid_FOV_circle,valid_background_rectangle,TC_port,TC_channel,IRCamColdframes_fittingpath,exJet,portpathdict,project,project_ana,heatflux_requestlist_path#,exJet_trans
15
16
17
18
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
Holger Niemann's avatar
Holger Niemann committed
19
20
21
import datetime
import urllib
import json
22
23
from PIL import Image
from io import BytesIO
24
from os.path import join
25
import matplotlib.pyplot as plt
26
27
try:          
    import threading
28
    import archivedb as AKF_2
29
30
31
32
    fastDL=True
except Exception as E:
    print(E)
    fastDL=False
Holger Niemann's avatar
Holger Niemann committed
33

34

35
try:
36
    config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
37
38
39
40
41
    FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0]   
    del FOV
except:
    config_path=""

42
43
44
45
46
47
48
49
50
51
52
53
54
55
def get_OP_by_program(program):
    try:    
        bla=int(program.split(".")[0])
        time_ns=int(TimeToNs([bla // 10000,bla % 10000 // 100,bla % 10000 % 100],[10,0,0,0]))
        return get_OP_by_time(time_ns)
    except Exception as E:
        print(E)
        return None
    
def get_OP_by_time(time_ns):
    dateOP=datetime.datetime.utcfromtimestamp(time_ns/1e9)
    if dateOP.year==2017:
        if dateOP.month>8 and dateOP.month<12:
            return "OP1.2a"
56
        elif dateOP.month==8 and dateOP.day>=28:
57
58
59
60
61
62
63
64
65
66
67
68
69
            return "OP1.2a"
        elif dateOP.month==12 and dateOP.day<8:
            return "OP1.2a"
        else:
            return None        
    elif dateOP.year==2018:
        return "OP1.2b"
    elif dateOP.year<=2016 and dateOP.year>=2015:
        if (dateOP.year==2016 and dateOP.month<=3) or (dateOP.year==2015 and dateOP.month==12):
            return "OP1.1"
        else:
            return None

70
def get_latest_version(stream,project="W7X",Test=False):
71
72
73
74
75
     """Find out latest version of given stream
     :param stream   url stream of interest
     :return int of version number of None if non-versioned or non-existing stream
     author: G. Schlisio, edit by holn
     """
76
77
78
79
80
     if Test:
         base="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
     else:
         base=archivepath    
     request =urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
     try:
         response = urllib.request.urlopen(request)
         d = json.loads(response.read().decode('utf-8'))
     except urllib.error.HTTPError as ex:
         msg = ex.read()
         raise RuntimeError(msg)
     # detect unversioned or non-existing stream
     if d["versionInfo"] == []:
         return None
     versions = []
     for i in d['versionInfo']:
         versions.append(i['number'])

     return max(versions)

Holger Niemann's avatar
Holger Niemann committed
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
def TimeToNs(date,time):
    """
    TimeToNs(date,time)
    (list)  date    [year,month,day]
    (list)  time    [hours,minutes,seconds,microseconds]
    """ 
    date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3])
    div=date_time-datetime.datetime(1970,1,1,0,0,0)
    nsdate=div.total_seconds()*1e9
    return nsdate

def read_program(timestampstart,timestamp_end=0,tol=60):
    """
    read_program()
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
    if timestamp_end!=0:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestamp_end+tol*1e9))
    else:
        jsonurl=program_url+"="+str(int(timestampstart-tol*1e9))+"&upto="+str(int(timestampstart+tol*1e9))
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw=res.read()
        res.close()
    except urllib.error.URLError as e:
        print('!get_program: Error opening URL')
        print(e)
        return False,0
    else:
        prog_string=prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl=prog_list['programs'][0]
        return True, pl

def read_restdb_old(request_url):
    """
    Reads JSON data from W7-X REST API
    Returns: 
        valid: access ok
        t: numpy-array of time
        signal: numpy-array of requested signals
137
    by H. Thomsen
Holger Niemann's avatar
Holger Niemann committed
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
    """
    try:
        res = urllib.request.urlopen(request_url)
    except urllib.error.URLError as e:
        print(e)
        return False, 0, -1
    else:
        signal_raw=res.read()
        res.close()
        signal_string=signal_raw.decode(encoding='UTF-8')
        signal_list = json.loads(signal_string)
        signal0=np.array(signal_list['values'])
        t=np.array(signal_list['dimensions'])
        return True, t, signal0

153
def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
Holger Niemann's avatar
Holger Niemann committed
154
155
156
    """
    download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
        time in ns
157
    Have to swap 11, 21 until correction in the database
Holger Niemann's avatar
Holger Niemann committed
158
    """    
159
    OP=IR_tools.get_OP_by_time(time_ns=time)
160
161
162
163
    if port == 21:
        port = 11
    elif port == 11:
        port = 21
164
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
165
    if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
166
        query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
167
    elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
Holger Niemann's avatar
Holger Niemann committed
168
169
170
171
        query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
        print("camera unknown, stopping here")
        raise Exception
172
    if version==0:
173
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM")
174
175
    if verbose>0:
        print("LUT V"+str(version)+" is used")
Holger Niemann's avatar
Holger Niemann committed
176
    #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
177
    LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
178
179
    if LUTpar[0]:
        LUTid=LUTpar[2][0]['structure'][query]
180
        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
181
        if LUTs[0]:
182
            LUTs=LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
Holger Niemann's avatar
Holger Niemann committed
183
184
185
186
            LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True,LUT
        else:
187
            print("Warning: unable to download the LUTs")
Holger Niemann's avatar
Holger Niemann committed
188
189
190
191
            del LUTpar, LUTs
            return False,0
    else:
        del LUTpar
192
        print("Warning: unable to find LUTs, check your request")
Holger Niemann's avatar
Holger Niemann committed
193
194
        return False,0

195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217

def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
    
    Read LUT V3 from local files for testing new calibration.
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
    OP = get_OP_by_time(this_time)
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
        with open(LUT_dir+"\\"+filename) as data_file:  
            jfile=json.load(data_file)
        LUT=np.array(jfile['LUT']).swapaxes(0,1)
        if verbose>0:
            print("succesfully loaded V3 LUT from local directory")
        return True, LUT
    except Exception as E:
        print("ERROR in loading V3 LUTs",E)
        return False, []
    
218
def download_NUC_by_program(port,program,exposure,version=0):
219
    prog=get_program_from_PID(program)
220
221
222
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
Holger Niemann's avatar
Holger Niemann committed
223
224
225
226
227
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_NUC_by_times(port,starttime,stoptime,exposure,version)
    else:
228
#    except:
Holger Niemann's avatar
Holger Niemann committed
229
230
231
        print("cannot find the program")
        return False,0,0

232
def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
233
    project="W7X"
234
    OP=get_OP_by_time(starttime)
235
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
Holger Niemann's avatar
Holger Niemann committed
236
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
237
    if version==0:
238
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        
        n=0
        nuctimes=[0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp']==exposure:
                nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i=NUCpar['structure']['offset_index']                
            n+=1 
        if nuctimes[0]!=0:
257
            NUC=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
258
259
260
261
262
263
264
265
266
267
268
            if NUC[0]:
                images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)                    
                return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
            else:
                print("NUC image for requested exposure time not found")
                return False, 0, 0
        else:
            print("NUC image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
269

270
def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
271
    prog=get_program_from_PID(program)
272
273
274
275
276
277
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
278
        return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose-1)
279
280
281
282
283
    else:
#    except:
        print("cannot find the program")
        return False,0,0
    
284
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
285
286
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if OP=="OP1.2a":
287
288
289
290
        t1=endtime
        t0=starttime
        prog=read_program(t1)
        program=prog[1]['id']
291
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):
292
            
293
294
295
296
297
298
299
300
301
302
303
304
305
306
            print("rebuilding coldframe")
            #use any way the rebuilt coldframe.
            sT = get_sensor_temp_by_program(port, program)[2][0]
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("unable to download reference frames")
            filestring = 'AEF' + str(port) + '_et' + str(t_exp)
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
307
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
            gain_error=0
            offset_error=0        
            return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        elif (port == 11) or (port == 21):
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
                raise Exception("unable to download reference frames")
            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
            if NUC_DL[0]==False:
                raise Exception("NUC was not found")
            else:
                ci = NUC_DL[1][2]
327
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1)
328
329
330
331
332
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)                                
            gain_error=0
            offset_error=0
333
            return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
        else:
            return download_NUC_by_times(port,starttime,endtime,t_exp)
    elif OP=="OP1.2b":
        gain_error=0
        offset_error=0
        gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
        if gotit:
            coldref=hotcold[1]
            hotref=hotcold[0]
        else:
            raise Exception("unable to download reference frames")
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
        exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
        if exist:
            cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
            for ele in coldframes:
                cold+=ele                
            cold=np.asarray(cold/len(coldframes),dtype=np.uint16)        
        else:
            return False,[0],[0]
354
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
355
356
357
358
359
360
361
362
363
364
        badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
        return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
    else:
        raise Exception("unknown Operation phase or NUC method not implemented for this OP")

def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0):
    """
    download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
    """
Holger Niemann's avatar
Holger Niemann committed
365
    gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(endtime-20e6))
366
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
367
368
369
370
371
372
373
374
    if gotit:
        expinds=np.where(texp_t==t_exp)[0]
        if len(expinds)==0:
            print("cannot find the exposure time in the given data")
            return False,[0],[0]                               
    else:
        print("exposure time not found")
        return False,[0],[0]
Holger Niemann's avatar
Holger Niemann committed
375
    gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(endtime-20e6))
376
    if gotitf:
Holger Niemann's avatar
Holger Niemann committed
377
        typiinds=np.where(values_f[expinds]==frametype)[0]
378
        if len(typiinds)>0:
Holger Niemann's avatar
Holger Niemann committed
379
380
381
            ref_t=[np.min(timef[expinds][typiinds]),np.max(timef[expinds][typiinds])]
#            print((ref_t[1]-ref_t[0])/1e9)
#            print(len(timef),len(timef[expinds]),len(timef[expinds][typiinds]))
382
383
384
385
386
387
388
        else:#okay the early data stuff or strange stuff
            print("frame type was not identified assuming that the first part is the cold and the second one the background")
            frametimes=time_t[expinds]
            diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
            turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
            if frametype==0:
                ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
Holger Niemann's avatar
Holger Niemann committed
389
                print((ref_t[1]-ref_t[0])/1e9)
390
391
392
            elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
                ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
Holger Niemann's avatar
Holger Niemann committed
393
                print((ref_t[1]-ref_t[0])/1e9)
394
395
396
397
398
399
400
            else:
                raise Exception("requested Frametype unknown and not implemented!")
    else:
        print("frametype not found")
        return False,[0],[0]
    t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
    t1date=t1date.isoformat()
Holger Niemann's avatar
Holger Niemann committed
401
    t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
402
403
    t0date=t0date.isoformat()
    if version==0:
404
405
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
406
    timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
Holger Niemann's avatar
Holger Niemann committed
407
    t_offset=timest0[-1][0]-time_t[0]
408
    if t_offset>10e6:
Holger Niemann's avatar
Holger Niemann committed
409
        print("time offset detected, try to correct this, offset is",t_offset)
410
411
412
413
414
415
    else:
        t_offset=0
#            print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
    return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset)
        
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
416
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
417
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
418
419
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
420
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon:        
        COLDtime=0
        HOTtime=0
        try:
            COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
        except:
            print("cold image for requested exposure time not found")
        try:
            HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
        except:
            print("hot image for requested exposure time not found")
        images=[]
        if HOTtime!=0:
442
            HOT=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
443
444
445
            if HOT[0]:
                images.append(HOT[2])
        if COLDtime!=0:
446
            COLD=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
447
448
449
450
451
452
453
454
455
456
            if COLD[0]:
                images.append(COLD[2])
                
        if HOT[0] and COLD[0] and len(images)==2:
            return True,images,['hot','cold']
        else:
            print("hot and cold image for requested exposure time not found")
            return False,0,0
    else:
        return False,0,0
Holger Niemann's avatar
Holger Niemann committed
457
    
458
def download_background_by_program(port,program,exposure,version=0):
459
460
461
    '''
    returned exist,time,frame
    '''
462
    prog=get_program_from_PID(program)
463
464
465
466
467
468
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
469
        return download_background_by_times(port,starttime,stoptime,exposure,version)
470
    else:#except:
Holger Niemann's avatar
Holger Niemann committed
471
472
473
        print("cannot find the program")
        return False,0,0
    
474
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
475
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
476
    stream=portpathdict[OP]["AEF"+str(port)]+"background_"
477
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
478
479
    if version==0:
        version=get_latest_version(stream+"DATASTREAM")
Holger Niemann's avatar
Holger Niemann committed
480
481
482
483
484
485
486
487
488
489
490
491
492
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
        print(e)
        goon=False
    if goon: 
        n=0
        backtimes=[0]
        for backpar in signal_list['values']:
            if backpar['structure']['Texp']==exposure:
493
                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
494
495
496
497
498
499
                    if backpar['structure']['filter']==camera_filter:
                        backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
                else:
                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
            n+=1
        if backtimes[0]!=0:
500
            backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
501
502
503
504
505
506
507
508
509
510
511
            if backdat[0]:                                    
                return backdat#[True,backdat[2]]
            else:
                print("background image for requested exposure time(, filter) not found")
                return False,0,0
        else:
            print("background image for requested exposure time(, filter) not found")
            return False,0,0
    else:
        return False,0,0

512
def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False):
513
514
    "OP1.2b function"
    exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version)
515
    camera=portpathdict["OP1.2b"]["AEF"+str(port)]    
516
517
518
519
520
521
522
523
524
    background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
    for frame in backgroundframes:
        background+=frame
    background=np.asarray(background/len(backgroundframes),dtype=np.uint16)
    if plot_it:
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image unnuced")
    if not camera.split("_")[0]=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
525
        background=apply_NUC([background],gain,offset)[0]
526
    if plot_it:        
Holger Niemann's avatar
Holger Niemann committed
527
        
528
529
530
531
532
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image nuced")
    return exist,btime[0],background

533
def download_raw_images_by_program(port,program,time_s=0,version=0,verbose=0):
534
535
536
537
538
539
540
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
541
542
543
        if type(time_s)==list or type(time_s)==np.ndarray:
            if len(time_s)>2 or time_s[0]>1000:
                raise Exception("invalid input for the timewindow!")
Holger Niemann's avatar
Holger Niemann committed
544
545
546
547
            tstart=time_s[0]
            tstop=time_s[1]
            if tstop<tstart:
                raise Exception("endtime before starttime")
548
            return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
549
        else:
Holger Niemann's avatar
Holger Niemann committed
550
            if time_s==0:
551
                return download_raw_images_by_times(port,starttime,stoptime,version,verbose=verbose-1)
Holger Niemann's avatar
Holger Niemann committed
552
            else:
553
                return download_raw_images_by_times(port,starttime,int(starttime+time_s*1e9),version,verbose=verbose-1)
554
    except:
555
556
        if verbose>0:
            print("cannot find the program")
Holger Niemann's avatar
Holger Niemann committed
557
558
        return False,0,0

559
def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
560
561
    OP=IR_tools.get_OP_by_time(time_ns=starttime)    
    if version==0:
562
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
563
564
565
566
567
568
569
570
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"  
    return download_images_by_times(larchivepath,starttime=starttime,stoptime=stoptime,version=version,intervalSize=intervalSize,verbose=verbose-1)
    
def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
    if '_raw' in larchivepath:
        typo=np.uint16          
    else:
        typo=np.float32
571
572
    if (stoptime-starttime)/intervalSize>1:        
        nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
573
574
        if verbose>0:
            print("timewindow to large, splitting into smaller fractions ("+str(nrinterv)+")",(stoptime-starttime)/1e9)
Holger Niemann's avatar
Holger Niemann committed
575
        intervalls=[]
576
        for i in range(nrinterv):
577
            intervalls.append(int(starttime-10+i*intervalSize))
Holger Niemann's avatar
Holger Niemann committed
578
        intervalls.append(stoptime)
579
580
581
        time=-1
        allimages=0
        success=False
582
        for i in range(nrinterv):
Holger Niemann's avatar
Holger Niemann committed
583
            try:
Holger Niemann's avatar
Holger Niemann committed
584
                res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1]))
Holger Niemann's avatar
Holger Niemann committed
585
                signal_list = json.loads(res.read().decode('utf-8'))
586
587
                res.close()                
                images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
588
#                signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
589
590
591
                t=np.array(signal_list['dimensions'])
                del signal_list
                if i==0:
592
                    allimages=images.copy()
Holger Niemann's avatar
Holger Niemann committed
593
594
                    time=t
                else:
595
                    allimages=allimages+images#np.append(ressignal,signal,axis=0)
Holger Niemann's avatar
Holger Niemann committed
596
                    time=np.append(time,t)
597
                del images,t
598
                success=True
Holger Niemann's avatar
Holger Niemann committed
599
            except urllib.error.URLError as e:
600
                
601
602
                if verbose>0:
                    print(e)                
603
        return success,time,allimages
Holger Niemann's avatar
Holger Niemann committed
604
605
606
607
608
    else:
        try:
            res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
609
            images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
610
#            signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
611
612
            t=np.array(signal_list['dimensions'])
            del signal_list
613
            return True, t, images
Holger Niemann's avatar
Holger Niemann committed
614
        except urllib.error.URLError as e:
615
616
            if verbose>0:
                print(e)
Holger Niemann's avatar
Holger Niemann committed
617
            return False, 0,-1
618

619
if fastDL:
620
    def download_raw_images_by_program_via_png(port,program,time_s=0,version=0,threads=1, verbose=0):
621
        prog=get_program_from_PID(program)
622
623
624
#        try:
#            t_program = AKF_2.get_program_from_to(program)
#            prog =AKF_2.get_program_list(t_program[0], t_program[1])
625
626
627
628
629
        if not prog[0]:
            if verbose>0:
                print("program not found, cannot download the data")
                return False,[0],[0]
        else:
630
631
            starttime=prog[1]['trigger']['1'][0]
            stoptime=prog[1]['trigger']['6'][0]
632
            OP=get_OP_by_time(starttime)
633
            if version==0:
634
635
                version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
            larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
636
            return download_images_by_time_via_png(larchivepath,starttime,stoptime,time_s=time_s,threads=threads,version=version,verbose=verbose-1)
637
            
638
    def download_images_by_time_via_png(larchivepath,starttime,stoptime,time_s=0,threads=1,version=0,verbose=0):            
639
            success=True            
Holger Niemann's avatar
Holger Niemann committed
640
641
642
643
644
645
646
647
            if type(time_s)==list:
                tstart=time_s[0]
                tstop=time_s[1]
                if tstop<tstart:
                    raise Exception("endtime before starttime")
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
                stdate=stdate.isoformat()
                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
648
649
                enddate=enddate.isoformat()
            else:
Holger Niemann's avatar
Holger Niemann committed
650
651
652
653
654
655
656
657
                stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
                stdate=stdate.isoformat()
                if time_s==0:
                    enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
                    enddate=enddate.isoformat()
                else:
                    enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)  
                    enddate=enddate.isoformat()
658
                #"2017-11-15 08:00:00"
659
660
661
            try:
                times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
            except Exception as E:
662
                print(larchivepath)
663
664
                raise Warning(E)
                return False,[0],[0]            
665
666
667
668
669
670
            time=[]
            images=[]
            lnt=len(times)
            if threads==1:
                for i in range(lnt):
                    ele=times[lnt-1-i]
671
                    imag=download_last_raw_image_by_time(larchivepath,ele[0]-10,ele[0]+10)
672
673
                    if imag[0]:
                        time.append(ele[0])
674
                        images.append(np.array(imag[1],dtype=np.uint16))
675
676
                    else:
                        success=False
677
                return success,np.array(time),images
678
679
680
681
682
683
684
685
686
687
            else:
                tim=[]
                for i in range(lnt):
                    tim.append(times[lnt-1-i][0])
                intervalls=[]
                intervalSize=int(lnt/threads)
                for i in range(threads):
                    intervalls.append(int(i*intervalSize))
                intervalls.append(lnt)
                jobs = []
688
                resultdict = []
689
                for i in range(threads):
690
691
                    if verbose>0:
                        print("Start Thread ",i+1)                
692
693
694
                    TH=download_images_thread(i,larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,version)
                    jobs.append(TH)
                    TH.start()
695
696
                for p in jobs:
                    p.join()
697
698
                if verbose>0:
                    print("all threads are done")
699
700
701
702
                order=[]
                for ele in resultdict:
                    order.append(ele[0])
                    if len(np.where(np.asarray(ele[1])==False)[0])>0:
703
                        success=False                
704
705
706
707
708
                times=[]#np.array(resultdict[order.index(0)][2])
                images=[]#np.array(resultdict[order.index(0)][3])
                for i in range(threads):
                    images=images+resultdict[order.index(i)][3]
                    times=times+resultdict[order.index(i)][2]
709
                    resultdict[order.index(i)]=[]
710
                del resultdict
711
                return success,np.array(times),images
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
      
    class download_images_thread(threading.Thread):
        def __init__(self, threadID, larchivepath,times,resultlist,version=0):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            self.resultlist.append([self.threadID,successes,time,images])
734
    
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
    class convert_raw_to_temp_thread(threading.Thread):
        def __init__(self,larchivepath,times,resultlist,threadID,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
        def run(self):
            images=[]
            time=[]
            successes=[]        
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            if self.give_ERROR:
                sucess,images,error_images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR)
767
            else:
768
769
770
771
772
773
774
                success,images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR)
            
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
            print(datetime.datetime.now(),"correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels)
            self.resultlist.append([self.threadID,successes,time,images])
775
    
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
    class NUC_raw_thread(threading.Thread):
        def __init__ (self,larchivepath,times,resultlist,threadID,version=0,background=0,LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,give_ERROR=False,FOV=0,badpixels=[]):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            if self.give_ERROR:
                images,error_images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
808
            else:
809
810
811
812
813
814
815
                images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
            
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
            print(datetime.datetime.now(),"correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels)
            self.resultlist.append([self.threadID,successes,time,images]) 
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845

def download_raw_FLIR_images_via_png(t1,t6,time_s=0,version=0,threads=1,verbose=0):    
    larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
    if type(time_s)==list:
        tstart=time_s[0]
        tstop=time_s[1]
        if tstop<tstart:
            raise Exception("endtime before starttime")
        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9+tstart)
        stdate=stdate.isoformat()
        enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+tstop)  
        enddate=enddate.isoformat()
    else:
        stdate=datetime.datetime.utcfromtimestamp((t1-100)/1e9)
        stdate=stdate.isoformat()
        if time_s==0:
            enddate=datetime.datetime.utcfromtimestamp(t6/1e9)        
            enddate=enddate.isoformat()
        else:
            enddate=datetime.datetime.utcfromtimestamp((t1)/1e9+time_s)  
            enddate=enddate.isoformat()
        #"2017-11-15 08:00:00"
    typ1_version=get_latest_version("QSR07_FLIR/AEK51_raw_DATASTREAM")
    if typ1_version==None:
        mode=1
    else:
        mode=2
    if mode==1:
        ### okay the data is stored probably as different sets (so recorded with FLIR software)
        timess,versions=get_FLIR_Pset_times(stdate,enddate,version)
846
847
        if verbose>0:
            print("found ",np.shape(timess[0]),np.shape(timess[1]),np.shape(timess[2]),np.shape(timess[3])," frames in the different sets")
848
849
850
851
        ### we not know the times for all presets for the program, now we have to download them, if times is not empty
        exist,time,frames,tsets=download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=verbose-1)            
    elif mode==2:##the data is stored only in one datastream, at least this datastream exist
        raise Exception("not implemented")
852
853
        #download data and downlaod LUT
        
854
855
856
857
858
859
860
861
862
863
864
865
866
#        time=[]
#        images=[]
#        lnt=len(times)
#        if threads==1:
#            for i in range(lnt):
#                ele=times[lnt-1-i]
#                imag=download_last_raw_image_by_time(port,ele[0]-10,ele[0]+10)
#                if imag[0]:
#                    time.append(ele[0])
#                    images.append(np.array(imag[1],dtype=np.uint16))
#                else:
#                    success=False
#            return success,np.array(time),images
867
    else:
868
869
870
871
        raise Exception("cannot identifie the way the searched data is stored!")                        

    return exist,time,frames,mode,tsets,versions

872
def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0):
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
    larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
    if version==0:
        versions=[]
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS0_DATASTREAM"))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS1_DATASTREAM"))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS2_DATASTREAM"))
        versions.append(get_latest_version("QSR07_FLIR/AEK51_raw_PS3_DATASTREAM"))
    else:
        versions=[version,version,version,version]
    timess=[]
    if version==0:##no version specified search for data in all versions            
        counter=-1
        for version0 in versions:
            counter+=1
            if version0>0:
                found=False
                while (version0>0 and not found):                    
                    try:            
                        times_0=AKF_2.get_time_intervals(larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw",stdate.replace("T"," "),enddate.replace("T"," "))#
                        found=True
                    except Exception as E:
894
                        print(E,larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw")
895
896
897
898
899
900
901
902
903
904
905
906
                        found=False
                        times_0=[]
                        version0=version0-1
                timess.append(times_0)
            else:
                timess.append([])
    else:
        counter=-1
        for version0 in versions:
            counter+=1
            try:            
                times_0=AKF_2.get_time_intervals(larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw",stdate.replace("T"," "),enddate.replace("T"," "))#
907
908
            except Exception as E:
                print(E,larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw")
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
                times_0=[]
            timess.append(times_0)
    return timess,versions

def download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=0):
    if len(timess[0])==len(timess[1]) and len(timess[2])==len(timess[3]) and len(timess[0])==0:
            exist=False
            time=[]
            frames=[]
            tsets=[]
    else:#timess is not empty, there is data in at least one Pset
        if verbose>0:
            print(datetime.datetime.now(),"data found, start the download")
        exist=True
        timesets=[]
        framess=[]
        success=True
        Psets=[]
        if threads==1:            
            counter=-1
            setn=-1
            for times in timess:                    
                setn+=1
                if times!=[]:
                    counter+=1
                    Psets.append(counter)
                    framess.append([])
                    timesets.append([])
                    lnt=len(times)                    
                    for i in range(lnt):                        
                        ele=times[lnt-1-i]
                        imag=download_last_raw_image_by_time(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw",ele[0]-10,ele[0]+10)
                        if imag[0]:
                            timesets[counter].append([ele[0],i,setn,counter])
                            framess[counter].append(np.array(imag[1],dtype=np.uint16))
                        else:
                            success=False
        else:#several threads, each for a preset
            tim=[]
            counter=-1
            setn=-1
            threads=0
            larchivepaths=[]
            versionss=[]
            for times in timess:                    
954
                times=np.array(times)
955
956
957
958
959
960
                setn+=1
                if times!=[]:
                    counter+=1
                    threads+=1
                    Psets.append(counter)
                    larchivepaths.append(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw")
961
                    tim.append(times[:,0])
962
963
                    versionss.append(versions[setn])
            jobs = []
964
            resultdict = []
965
966
967
            for i in range(threads):
                if verbose>0:
                    print("Start Thread ",i+1)                
968
969
970
                TH=download_images_thread(i,larchivepaths[i],tim[i],resultdict,versionss[i])
                jobs.append(TH)
                TH.start()            
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
            for p in jobs:
                p.join()
            if verbose>0:
                print("all threads are done")
            order=[]
            for ele in resultdict:
                order.append(ele[0])
                if len(np.where(np.asarray(ele[1])==False)[0])>0:
                    success=False                
            for i in range(threads):
                framess.append(resultdict[order.index(i)][3])
                timeline=[]
                for j in range(len(resultdict[order.index(i)][2])):
                    timeline.append([resultdict[order.index(i)][2][j],j,Psets[i],i])
                timesets.append(timeline)
                resultdict[order.index(i)]=[]
            del resultdict,timeline
        if verbose>0:
            print(datetime.datetime.now(),"download finished, successfull",success)
        if success:
            del timess
            ### okay the have now the data from all presets, now we have to sort the frames into one array
            time=[]
            frames=[]                
            dummyT=[]
            tsets=[]
            for tim in timesets:
                dummyT=dummyT+tim
            dummyT.sort()
            for ele in dummyT:
For faster browsing, not all history is shown. View entire blame