downloadversionIRdata.py 165 KB
Newer Older
Holger Niemann's avatar
Holger Niemann committed
1
2
3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
4
updated on Tue Aug 21 10:20:00 2018
5
last update on Fr Nov 23 15:37:00 2018
Holger Niemann's avatar
Holger Niemann committed
6

7
Version: 3.2.4
8
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
Holger Niemann's avatar
Holger Niemann committed
9
10
11
12
@author: holn
"""

import numpy as np
13
import IR_image_tools as IR_tools
14
15
16
17
18
from IR_config_constants import archivepath,portcamdict,camlimdict, \
                                valid_FOV_circle,valid_background_rectangle, \
                                TC_port,TC_channel,IRCamColdframes_fittingpath, \
                                exJet,portpathdict,project,project_ana, \
                                heatflux_requestlist_path#,exJet_trans
19
20
21
22
#try:
#    import W7Xrest.read_restdb as AKF_1
#except:
#    import read_restdb as AKF_1
Holger Niemann's avatar
Holger Niemann committed
23
24
25
import datetime
import urllib
import json
26
27
from PIL import Image
from io import BytesIO
28
from os.path import join
29
import matplotlib.pyplot as plt
30
31
try:          
    import threading
32
    import archivedb as AKF_2
33
34
35
36
    fastDL=True
except Exception as E:
    print(E)
    fastDL=False
Holger Niemann's avatar
Holger Niemann committed
37

38

39
try:
40
    config_path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Software\\QI-RealTime\\1.0.0\\release\\QIR-IrAnalysis\\Config\\Thermal calibration\\"
41
42
43
44
45
    FOV=plt.imread(config_path+portcamdict['OP1.2a']['AEF'+str(10)]+"\\"+"AEF"+str(10)+"_FOV.png")[:,:,0]   
    del FOV
except:
    config_path=""

46

47
def get_latest_version(stream,project="W7X",Test=False,t_from=None,t_to=None,program=None):
48
    
49
50
51
52
53
     """Find out latest version of given stream
     :param stream   url stream of interest
     :return int of version number of None if non-versioned or non-existing stream
     author: G. Schlisio, edit by holn
     """
54
55
56
57
     if Test:
         base="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
     else:
         base=archivepath    
58
59
60
61
62
63
64
65

    
     if program==None and t_from==None and t_to==None:         
         request =urllib.request.Request(base +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})            
     else:
         if program is not None:
             prog=get_program_from_PID(program)
             if prog[0]:
66
                 t_from=prog[1]['trigger']['0'][0]
67
68
69
70
71
72
                 t_to=prog[1]['trigger']['6'][0]
         elif t_from is not None and t_to==None:
             t_to=int(t_from+100e9)
         elif t_from==None and t_to is not None:
             t_from=int(t_to-10e9)
         request =urllib.request.Request(base +project+"/" + stream + "/_versions.json?from="+str(t_from)+"&upto="+str(t_to), headers={"Accept": "application/json"})
73
74
75
76
77
78
     try:
         response = urllib.request.urlopen(request)
         d = json.loads(response.read().decode('utf-8'))
     except urllib.error.HTTPError as ex:
         msg = ex.read()
         raise RuntimeError(msg)
79
80
     except Exception as E:
         raise RuntimeError(E)
81
82
     else:
         response.close()
83
84
85
86
87
88
89
90
91
     # detect unversioned or non-existing stream
     if d["versionInfo"] == []:
         return None
     versions = []
     for i in d['versionInfo']:
         versions.append(i['number'])

     return max(versions)

Holger Niemann's avatar
Holger Niemann committed
92
93
94
95
96
97
98
99
100
101
102
def TimeToNs(date,time):
    """
    TimeToNs(date,time)
    (list)  date    [year,month,day]
    (list)  time    [hours,minutes,seconds,microseconds]
    """ 
    date_time=datetime.datetime(date[0],date[1],date[2],time[0],time[1],time[2],time[3])
    div=date_time-datetime.datetime(1970,1,1,0,0,0)
    nsdate=div.total_seconds()*1e9
    return nsdate

103
def read_program(timestamp_start,timestamp_end=0,tol=60):
Holger Niemann's avatar
Holger Niemann committed
104
105
106
107
    """
    read_program()
    """
    program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from'
108
    if timestamp_end==0:
109
110
111
112
        timestamp_end = timestamp_start
    jsonurl = '{0}={1}&upto={2}'.format(program_url,
                                        int(timestamp_start-tol*1e9),
                                        int(timestamp_end+tol*1e9))
Holger Niemann's avatar
Holger Niemann committed
113
114
115
116
117
    try:
        res = urllib.request.urlopen(jsonurl)
        prog_raw=res.read()
        res.close()
    except urllib.error.URLError as e:
118
        print('read_program: Error opening URL')
Holger Niemann's avatar
Holger Niemann committed
119
120
        print(e)
        return False,0
121
122
123
    except Exception as e:
        print('read_program: Error opening URL',e)
        return False,0
Holger Niemann's avatar
Holger Niemann committed
124
125
126
127
128
129
    else:
        prog_string=prog_raw.decode(encoding='UTF-8')
        prog_list = json.loads(prog_string)
        pl=prog_list['programs'][0]
        return True, pl

130

Holger Niemann's avatar
Holger Niemann committed
131

132
def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
Holger Niemann's avatar
Holger Niemann committed
133
134
135
    """
    download_LUT(camera,port,time,exposure=0,emissivity=0,camera_filter=0,version=1):
        time in ns
136
    Have to swap 11, 21 until correction in the database
Holger Niemann's avatar
Holger Niemann committed
137
    """    
138
    OP=IR_tools.get_OP_by_time(time_ns=time)
139
140
141
142
    if port == 21:
        port = 11
    elif port == 11:
        port = 21
143
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"LUT_"
144
    if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
145
        query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
146
    elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
Holger Niemann's avatar
Holger Niemann committed
147
148
        query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
    else:
149
        print("download_LUT: Error! Camera unknown, stopping here.")
Holger Niemann's avatar
Holger Niemann committed
150
        raise Exception
151
    if version==0:
152
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"LUT_DATASTREAM",t_from=time)
153
    if verbose>0:
154
        print("download_LUT: LUT V"+str(version)+" is used")
Holger Niemann's avatar
Holger Niemann committed
155
    #time=int(fu.TimeToNs([2017,9,26],[8,0,0,0]))
156
    LUTpar=read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
157
158
    if LUTpar[0]:
        LUTid=LUTpar[2][0]['structure'][query]
159
        LUTs=read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
Holger Niemann's avatar
Holger Niemann committed
160
        if LUTs[0]:
161
            LUTs=LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
Holger Niemann's avatar
Holger Niemann committed
162
163
164
165
            LUT=[LUTs[0],LUTs[LUTid],LUTs[LUTid+1]]
            del LUTpar, LUTs
            return True,LUT
        else:
166
            print("download_LUT: Warning! unable to download the LUTs")
Holger Niemann's avatar
Holger Niemann committed
167
168
169
170
            del LUTpar, LUTs
            return False,0
    else:
        del LUTpar
171
        print("download_LUT: Warning! unable to find LUTs, check your request")
Holger Niemann's avatar
Holger Niemann committed
172
173
        return False,0

174
175
176
177
178
179
180
181

def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_steel=0.31, verbose=0):
    """
    read_LUT_from_file(port, time, t_exp, emissivity, cfilter, verbose=verbose-1)
    
    Read LUT V3 from local files for testing new calibration.
    """
    LUT_dir = '\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\testingLUT'
182
    OP = IR_tools.get_OP_by_time(time_ns=this_time)
183
184
185
186
187
188
189
190
    cam_name = portcamdict[OP]['AEF{0}'.format(port)]
    filename = '{0}_thermal_LUT_filter_{1}_texp_{2}us_emiss_{3:.2f}.json'.format(cam_name, cfilter, t_exp, emissivity)
#    data = json.loads(join(LUT_dir, filename).decode('utf-8'))
    try:
        with open(LUT_dir+"\\"+filename) as data_file:  
            jfile=json.load(data_file)
        LUT=np.array(jfile['LUT']).swapaxes(0,1)
        if verbose>0:
191
            print("read_LUT_from_file: succesfully loaded V3 LUT from local directory")
192
193
        return True, LUT
    except Exception as E:
194
        print("read_LUT_from_file: ERROR in loading V3 LUTs",E)
195
196
        return False, []
    
197
def download_NUC_by_program(port,program,exposure,version=0):
198
    prog=get_program_from_PID(program)
199
200
201
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
Holger Niemann's avatar
Holger Niemann committed
202
203
204
205
206
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
        return download_NUC_by_times(port,starttime,stoptime,exposure,version)
    else:
207
#    except:
208
        print("download_NUC_by_program: cannot find the program")
Holger Niemann's avatar
Holger Niemann committed
209
210
        return False,0,0

211
def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
212
    project="W7X"
213
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
214
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
Holger Niemann's avatar
Holger Niemann committed
215
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
216
    if version==0:
217
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM",t_from=starttime)
Holger Niemann's avatar
Holger Niemann committed
218
219
220
221
222
223
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
224
        print('download_NUC_by_times: Error! ', e)
Holger Niemann's avatar
Holger Niemann committed
225
        goon=False
226
227
228
    except Exception as e:
        print('download_NUC_by_times: Error! ', e)
        goon=False
Holger Niemann's avatar
Holger Niemann committed
229
230
231
232
233
234
235
236
237
238
    if goon:        
        n=0
        nuctimes=[0]
        for NUCpar in signal_list['values']:
            if NUCpar['structure']['Texp']==exposure:
                nuctimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
#                gain_i=NUCpar['structure']['gain_index']
                offset_i=NUCpar['structure']['offset_index']                
            n+=1 
        if nuctimes[0]!=0:
239
            NUC=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
240
241
242
243
            if NUC[0]:
                images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)                    
                return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
            else:
244
                print("download_NUC_by_times: NUC image for requested exposure time not found")
Holger Niemann's avatar
Holger Niemann committed
245
246
                return False, 0, 0
        else:
247
            print("download_NUC_by_times: NUC image for requested exposure time not found")
Holger Niemann's avatar
Holger Niemann committed
248
249
250
            return False,0,0
    else:
        return False,0,0
251

252
def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
253
    prog=get_program_from_PID(program)
254
255
256
257
258
259
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
260
        return get_NUC_by_times(port,starttime,stoptime,exposure,version=0,verbose=verbose)
261
262
    else:
#    except:
263
        print("get_NUC_by_program: Error! cannot find the program")
264
265
        return False,0,0
    
266
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
267
268
269
270
271
    """
    Loads NUC elements (gain, offset cold image, bad pixels) for an IR camera
    in a given port and time interval. Depending on time and camera, reconstruct
    teh cold frame and/or compute the gain,offset from the hot and cold frame.
    """
272
273
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
    if OP=="OP1.2a":
274
275
276
277
        t1=endtime
        t0=starttime
        prog=read_program(t1)
        program=prog[1]['id']
278
        if (port == 31) or (port == 21 and float(program[4:]) > 1110):
279
280
            if verbose>0:
                print("get_NUC_by_times: rebuilding coldframe")
281
282
283
284
285
286
287
288
            #use any way the rebuilt coldframe.
            sT = get_sensor_temp_by_program(port, program)[2][0]
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
289
                raise Exception("get_NUC_by_times: unable to download reference frames")
290
291
292
293
            filestring = 'AEF' + str(port) + '_et' + str(t_exp)
            amap = np.load(join(IRCamColdframes_fittingpath, filestring + '_a.npy'))
            bmap = np.load(join(IRCamColdframes_fittingpath, filestring + '_b.npy'))
            cirebuild = IR_tools.reconstruct_coldframe(t_exp, sT, amap, bmap, coldref)
Peter Drewelow's avatar
Peter Drewelow committed
294
295
            if verbose>0:
                print('get_NUC_by_times: calculate gain, offset and bad pixels')
296
            gain, offset = IR_tools.calculate_gain_offset_image(cirebuild, None, coldref, hotref,verbose=verbose-1)
297
298
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
299
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)
300
301
302
303
            gain_error=0
            offset_error=0        
            return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
        elif (port == 11) or (port == 21):
Peter Drewelow's avatar
Peter Drewelow committed
304
305
            if verbose>0:
                print('get_NUC_by_times: downloading NUC')
306
307
308
309
310
311
    #                        coldref, hotref = IR_tools.load_ref_images('AEF' + str(port), t_exp)
            gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
            if gotit:
                coldref=hotcold[1]
                hotref=hotcold[0]
            else:
312
                raise Exception("get_NUC_by_times: unable to download reference frames")
313
314
            NUC_DL=download_NUC_by_times(port,t0,t1,t_exp,version)
            if NUC_DL[0]==False:
315
                raise Exception("get_NUC_by_times: NUC was not found")
316
317
            else:
                ci = NUC_DL[1][2]
318
            gain, offset = IR_tools.calculate_gain_offset_image(ci, None, coldref, hotref,verbose=verbose-1)
319
320
            gain[gain == np.inf] = 0
            offset[offset == np.inf] = 0
321
            badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)                                
322
323
            gain_error=0
            offset_error=0
324
            return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
325
        else:
Peter Drewelow's avatar
Peter Drewelow committed
326
327
            if verbose>0:
                print('get_NUC_by_times: downloading NUC')
328
329
330
331
332
333
334
335
336
            return download_NUC_by_times(port,starttime,endtime,t_exp)
    elif OP=="OP1.2b":
        gain_error=0
        offset_error=0
        gotit,hotcold,describtion=download_hot_cold_reference_by_times(port,t_exp)
        if gotit:
            coldref=hotcold[1]
            hotref=hotcold[0]
        else:
337
            raise Exception("get_NUC_by_times: unable to download reference frames")
338
        ### get the cold frame: 1. get the metastream frametype and exposuretime and check in it for the positions, if -1 type is unknown, when if needs to be identified in a different way
339
        exist,ctime,coldframes=download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0,verbose=verbose-1)
340
341
342
343
344
345
346
        if exist:
            cold=np.zeros(np.shape(coldframes[0]),dtype=np.uint64)
            for ele in coldframes:
                cold+=ele                
            cold=np.asarray(cold/len(coldframes),dtype=np.uint16)        
        else:
            return False,[0],[0]
347
        gain, offset = IR_tools.calculate_gain_offset_image(cold, None, coldref, hotref,verbose=verbose-1)
348
        badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10,verbose=verbose-1)
349
350
        return True,[gain,offset,cold,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
    else:
351
        raise Exception("get_NUC_by_times: unknown Operation phase or NUC method not implemented for this OP")
352

353
def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0,verbose=0):
354
355
356
357
    """
    download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametype=0,version=0)
    frametype: 0 for closed shutter frames (cold), 1 for open shutter frames (background)
    """
Holger Niemann's avatar
Holger Niemann committed
358
    gotit,time_t,texp_t=get_exposure_by_times(port,int(starttime-100),int(endtime-20e6))
359
    OP=IR_tools.get_OP_by_time(time_ns=endtime)
360
361
362
    if gotit:
        expinds=np.where(texp_t==t_exp)[0]
        if len(expinds)==0:
363
            print("download_calibration_raw_files_by_time: Error! cannot find the exposure time in the given data")
364
365
            return False,[0],[0]                               
    else:
366
        print("download_calibration_raw_files_by_time: Error! exposure time not found")
367
        return False,[0],[0]
Holger Niemann's avatar
Holger Niemann committed
368
    gotitf,timef,values_f=get_frametype_by_times(port,int(starttime-100),int(endtime-20e6))
369
    if gotitf:
Holger Niemann's avatar
Holger Niemann committed
370
        typiinds=np.where(values_f[expinds]==frametype)[0]
371
        if len(typiinds)>0:
Holger Niemann's avatar
Holger Niemann committed
372
373
374
            ref_t=[np.min(timef[expinds][typiinds]),np.max(timef[expinds][typiinds])]
#            print((ref_t[1]-ref_t[0])/1e9)
#            print(len(timef),len(timef[expinds]),len(timef[expinds][typiinds]))
375
        else:#okay the early data stuff or strange stuff
376
377
            if verbose>0:
                print("download_calibration_raw_files_by_time: frame type was not identified assuming that the first part is the cold and the second one the background")
378
379
380
381
382
            frametimes=time_t[expinds]
            diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
            turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
            if frametype==0:
                ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
Holger Niemann's avatar
Holger Niemann committed
383
                print((ref_t[1]-ref_t[0])/1e9)
384
385
386
            elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
                ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
Holger Niemann's avatar
Holger Niemann committed
387
                print((ref_t[1]-ref_t[0])/1e9)
388
            else:
389
                raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
390
    else:
391
        print("download_calibration_raw_files_by_time: Error! frametype not found")
392
393
394
395
396
397
398
399
400
401
402
403
404
        frametimes=time_t[expinds]
        diftime=[frametimes[i]-frametimes[i-1] for i in range(1,len(frametimes))]
        turnpoint=np.where(np.asarray(diftime)>11e6)[0][0]
        if frametype==0:
            ref_t=[np.min(frametimes[0:turnpoint+1]),np.max(frametimes[0:turnpoint+1])]
            print((ref_t[1]-ref_t[0])/1e9)
        elif frametype==1:
#                print(len(frametimes[turnpoint+1:]),len(frametimes[0:turnpoint+1]),len(frametimes))                
            ref_t=[np.min(frametimes[turnpoint+1:]),np.max(frametimes[turnpoint+1:])]
            print((ref_t[1]-ref_t[0])/1e9)
        else:
            raise Exception("download_calibration_raw_files_by_time: requested Frametype unknown and not implemented!")
#        return False,[0],[0]
405
406
    t1date=datetime.datetime.utcfromtimestamp((endtime-100)/1e9)
    t1date=t1date.isoformat()
Holger Niemann's avatar
Holger Niemann committed
407
    t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
408
409
    t0date=t0date.isoformat()
    if version==0:
410
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=endtime)
411
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
412
    timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
Holger Niemann's avatar
Holger Niemann committed
413
    t_offset=timest0[-1][0]-time_t[0]
414
    if t_offset>10e6:
415
416
        if verbose>0:
            print("download_calibration_raw_files_by_time: time offset detected, try to correct this, offset is",t_offset)
417
418
419
    else:
        t_offset=0
#            print("starttime frames:",np.min(timest0),"starttime metachannels:",time_t[0],"offset",t_offset)
420
    return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset,verbose=verbose-1)
421
422
        
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
423
424
425
426
    """
    Loads the most recent hot and cold calibration frames for a starttime.
    Uses first calibration frames if time is not defined.
    """
427
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
428
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
429
430
#    NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
    if version==0:
431
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime)
432
    try:
433
434
        path_string = larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9))
        res = urllib.request.urlopen(path_string)
435
436
437
438
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
439
        print('download_hot_cold_reference_by_times: Error! ',e)
440
        goon=False
441
442
443
    except Exception as e:
        print('download_hot_cold_reference_by_times: Error! ',e)
        goon=False
444
445
446
447
448
449
    if goon:        
        COLDtime=0
        HOTtime=0
        try:
            COLDtime=signal_list['values'][0]['structure']['cold_'+str(int(exposure))+'us']
        except:
450
            print("download_hot_cold_reference_by_times: cold image for requested exposure time not found")
451
452
453
        try:
            HOTtime=signal_list['values'][0]['structure']['hot_'+str(int(exposure))+'us']
        except:
454
            print("download_hot_cold_reference_by_times: hot image for requested exposure time not found")
455
456
        images=[]
        if HOTtime!=0:
457
            HOT=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
458
459
460
            if HOT[0]:
                images.append(HOT[2])
        if COLDtime!=0:
461
            COLD=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
462
463
464
465
466
467
            if COLD[0]:
                images.append(COLD[2])
                
        if HOT[0] and COLD[0] and len(images)==2:
            return True,images,['hot','cold']
        else:
468
            print("download_hot_cold_reference_by_times: Error! hot and cold image for requested exposure time not found")
469
470
471
            return False,0,0
    else:
        return False,0,0
Holger Niemann's avatar
Holger Niemann committed
472
    
473
def download_background_by_program(port,program,exposure,version=0):
474
475
476
    '''
    returned exist,time,frame
    '''
477
    prog=get_program_from_PID(program)
478
479
480
481
482
483
#    try:
#        t_program = AKF_2.get_program_from_to(program)
#        prog =AKF_2.get_program_list(t_program[0], t_program[1])
    if prog[0]:
        starttime=prog[1]['trigger']['0'][0]
        stoptime=prog[1]['trigger']['1'][0]
484
        return download_background_by_times(port,starttime,stoptime,exposure,version)
485
    else:#except:
486
        print("download_background_by_program: Error! cannot find the program {0}".format(program))
Holger Niemann's avatar
Holger Niemann committed
487
488
        return False,0,0
    
489
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
490
    OP=IR_tools.get_OP_by_time(time_ns=starttime)
491
    stream=portpathdict[OP]["AEF"+str(port)]+"background_"
492
    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
493
    if version==0:
494
        version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
Holger Niemann's avatar
Holger Niemann committed
495
496
497
498
499
500
    try:
        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
        signal_list = json.loads(res.read().decode('utf-8'))
        res.close()
        goon=True
    except urllib.error.URLError as e:
501
        print('download_background_by_times: Error! ',e)
Holger Niemann's avatar
Holger Niemann committed
502
        goon=False
503
504
505
    except Exception as e:
        print('download_background_by_times: Error! ',e)
        goon=False
Holger Niemann's avatar
Holger Niemann committed
506
507
508
509
510
    if goon: 
        n=0
        backtimes=[0]
        for backpar in signal_list['values']:
            if backpar['structure']['Texp']==exposure:
511
                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
512
513
514
515
516
517
                    if backpar['structure']['filter']==camera_filter:
                        backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
                else:
                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
            n+=1
        if backtimes[0]!=0:
518
            backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
Holger Niemann's avatar
Holger Niemann committed
519
520
521
            if backdat[0]:                                    
                return backdat#[True,backdat[2]]
            else:
522
                print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
Holger Niemann's avatar
Holger Niemann committed
523
524
                return False,0,0
        else:
525
            print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
Holger Niemann's avatar
Holger Niemann committed
526
527
528
529
            return False,0,0
    else:
        return False,0,0

530
531
532
533
534
def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False,verbose=0):
    """
    OP1.2b function
    """
    exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version,verbose=verbose-1)
535
536
537
    camera=portcamdict["OP1.2b"]["AEF"+str(port)]  
    if verbose>50:
        print("camera is",camera)
538
539
540
541
542
543
544
545
546
    background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
    for frame in backgroundframes:
        background+=frame
    background=np.asarray(background/len(backgroundframes),dtype=np.uint16)
    if plot_it:
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image unnuced")
    if not camera.split("_")[0]=="Infratec":
Holger Niemann's avatar
Holger Niemann committed
547
        background=apply_NUC([background],gain,offset)[0]
548
549
550
551
552
553
    if plot_it:        
        plt.figure()
        plt.imshow(background,vmin=np.median(background)-500,vmax=np.median(background)+500)
        plt.title("background image nuced")
    return exist,btime[0],background

554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587

def get_NUCed_coldframe_by_program(port, program, exposure=None, version=0, plot_it=False, verbose=0):
    """ 
       Load the raw cold refences frame taken bofore this program and NUC it with 
       the NUC of the previous program (or next one, if the previous does not exist).

       INPUT
       -----
           port: int
               number of camera AEF port (e.g. 10, 11, 20,...)
           program: str
               experiment program identifier as a string of format 'DATE.PROG_NO',
               e.g. '20180904.015'
           exposure: float, optional
               camera exposure time in us
               (OPTIONAL: default is to take the exposure time of the first data frame)
           version: int, optional
               calibration version to be used
               (OPTIONAL: default is 0)
           plot_it: bool, optional
               switch of whether to plot intermediate results or not
               (OPTIONAL: deafult is NOT to plot)
           verbose: integer, optional
               feedback level (details of print messages)
               (OPTIONAL: if not provided, only ERROR output)
              
       RESULT
       ------
           exist: bool
               indicator, of coldframe could be found
           coldframe: numpy array
               NUCed cold frame
    """
    if exposure is None:
588
589
590
591
592
593
        try:
            exist, t, exposure = get_exposure_by_program(port, program, version=version)
        except:
            print('get_NUCed_coldframe_by_program: ERROR! no exposure time found!')
            return False, []
            
594
595
596
597
598
        t_exp = exposure[0]
    else:
        if verbose>0:
            print('get_NUCed_coldframe_by_program: using exposure time {0}'.format(exposure))
        t_exp = exposure
599
600
601
602
603
604
    try:
        exist, data, desc = get_NUC_by_program(port, program, t_exp, version=version, verbose=verbose-1)
    except:
        print('get_NUCed_coldframe_by_program: ERROR! no cold frame found!')
        return False, []
    
605
606
607
608
609
610
611
612
613
614
615
616
617
    if exist:
        cold_raw = data[2]
        porg_nr = int(program.split('.')[1])
        
        ref_program = '{0}.{1:03d}'.format(program.split('.')[0], porg_nr-1)
        exist2, data, desc = get_NUC_by_program(port, ref_program, t_exp, version=version, verbose=verbose-1)
        if exist2:
            coldframe = apply_NUC([cold_raw],data[0],data[1])[0]
            if verbose>0:
                print('get_NUCed_coldframe_by_program: using NUC of program {0}'.format(ref_program))
        else:
            ref_program = '{0}.{1:03d}'.format(program.split('.')[0], porg_nr+1)
            if verbose>0:
618
                print('get_NUCed_coldframe_by_program: program {0} has no predecessor.\n   --> using NUC of {1}'.format(program, ref_program))
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
            exist2, data, desc = get_NUC_by_program(port, ref_program, t_exp, version=version, verbose=verbose-1)
            if exist2:
                coldframe = apply_NUC([cold_raw],data[0],data[1])[0]
            else:
                print('get_NUCed_coldframe_by_program: WARNING! no adjacent program found! --> cold frame is not NUCed')
                coldframe = cold_raw
        
        if plot_it:
            plt.figure(figsize=[8,4])
            plt.subplot(1,2,1)
            plt.imshow(cold_raw, vmin=np.percentile(cold_raw, 1), vmax=np.percentile(cold_raw, 99) )
            plt.colorbar()
            plt.title('raw cold frame\nof {0}'.format(program))
            plt.subplot(1,2,2)
            plt.imshow(coldframe, vmin=np.percentile(coldframe, 1), vmax=np.percentile(coldframe, 99) )
            plt.colorbar()
            plt.title('NUCed based on\ngain/offset from {0}'.format(ref_program))
            plt.tight_layout()
            plt.show()
    else:
        print('get_NUCed_coldframe_by_program: ERROR! No cold frame found for program {0}!'.foramt(program))
        coldframe = []
    
    return exist, coldframe


645
646
647
def download_raw_images_by_program(port,program,time_window=0,version=0,verbose=0):
    """
    """
648
649
650
651
652
653
654
#    prog=AKF_1.get_program_from_PID(program)
    try:
        t_program = AKF_2.get_program_from_to(program)
        prog =AKF_2.get_program_list(t_program[0], t_program[1])
#    if prog[0]:
        starttime=prog[0]['trigger']['1'][0]
        stoptime=prog[0]['trigger']['6'][0]
655
656
657
658
659
        if type(time_window)==list or type(time_window)==np.ndarray:
            if len(time_window)>2 or time_window[0]>1000:
                raise Exception("download_raw_images_by_program: invalid input for the time_window!")
            tstart=time_window[0]
            tstop=time_window[1]
Holger Niemann's avatar
Holger Niemann committed
660
            if tstop<tstart:
661
                raise Exception("download_raw_images_by_program: endtime before starttime")
662
            return download_raw_images_by_times(port,int(starttime+tstart*1e9),int(starttime+tstop*1e9),version,verbose=verbose)
Holger Niemann's avatar
Holger Niemann committed
663
        else:
664
            if time_window==0:
665
                return download_raw_images_by_times(port,starttime,stoptime,version,verbose=verbose)
Holger Niemann's avatar
Holger Niemann committed
666
            else:
667
                return download_raw_images_by_times(port,starttime,int(starttime+time_window*1e9),version,verbose=verbose)
668
    except:
669
        print("download_raw_images_by_program: Error! cannot find the program")
Holger Niemann's avatar
Holger Niemann committed
670
671
        return False,0,0

672
def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
673
674
    """
    """
675
676
    OP=IR_tools.get_OP_by_time(time_ns=starttime)    
    if version==0:
677
        version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime)
678
    larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"  
679
    return download_images_by_times(larchivepath,starttime=starttime,stoptime=stoptime,version=version,intervalSize=intervalSize,verbose=verbose)
680
681
    
def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
682
683
    """
    """
684
685
686
687
    if '_raw' in larchivepath:
        typo=np.uint16          
    else:
        typo=np.float32
688
689
    if (stoptime-starttime)/intervalSize>1:        
        nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
690
        if verbose>0:
691
            print("download_images_by_times: time_window to large, splitting {0}s interval into {1} smaller fractions".format((stoptime-starttime)/1e9,nrinterv))
Holger Niemann's avatar
Holger Niemann committed
692
        intervalls=[]
693
        for i in range(nrinterv):
694
            intervalls.append(int(starttime-10+i*intervalSize))
Holger Niemann's avatar
Holger Niemann committed
695
        intervalls.append(stoptime)
696
697
698
        time=-1
        allimages=0
        success=False
699
        for i in range(nrinterv):
Holger Niemann's avatar
Holger Niemann committed
700
            try:
701
702
703
704
                url = larchivepath+"/_signal.json?from="+str(intervalls[i])+"&upto="+str(intervalls[i+1])
                if verbose>2:
                    print("download_images_by_times: reading from\n   {0}".format(url))
                res = urllib.request.urlopen(url)
Holger Niemann's avatar
Holger Niemann committed
705
                signal_list = json.loads(res.read().decode('utf-8'))
706
707
                res.close()                
                images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
708
#                signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
709
710
711
                t=np.array(signal_list['dimensions'])
                del signal_list
                if i==0:
712
                    allimages=images.copy()
Holger Niemann's avatar
Holger Niemann committed
713
714
                    time=t
                else:
715
                    allimages=allimages+images#np.append(ressignal,signal,axis=0)
Holger Niemann's avatar
Holger Niemann committed
716
                    time=np.append(time,t)
717
                del images,t
718
                success=True
Holger Niemann's avatar
Holger Niemann committed
719
            except urllib.error.URLError as e:
720
                print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e))                
721
722
            except Exception as e:
                print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e))   
723
        return success,time,allimages
Holger Niemann's avatar
Holger Niemann committed
724
725
    else:
        try:
726
727
            if verbose>100:
                print(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))
728
            res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime-10)+"&upto="+str(stoptime))            
Holger Niemann's avatar
Holger Niemann committed
729
730
            signal_list = json.loads(res.read().decode('utf-8'))
            res.close()
731
            images=[np.array(ele, dtype=typo) for ele in signal_list['values']]
732
#            signal=np.array(signal_list['values'],dtype=np.uint16)
Holger Niemann's avatar
Holger Niemann committed
733
734
            t=np.array(signal_list['dimensions'])
            del signal_list
735
            return True, t, images
Holger Niemann's avatar
Holger Niemann committed
736
        except urllib.error.URLError as e:
737
            print('download_images_by_times: Error! ',e)
Holger Niemann's avatar
Holger Niemann committed
738
            return False, 0,-1
739
740
741
        except Exception as e:
            print('download_images_by_times: Error! ',e)
            return False, 0,-1
742

743
if fastDL:
744
745
746
    def download_raw_images_by_program_via_png(port,program,time_window=0,version=0,threads=1,verbose=0):
        """
        """
747
        prog=get_program_from_PID(program)
748
749
750
#        try:
#            t_program = AKF_2.get_program_from_to(program)
#            prog =AKF_2.get_program_list(t_program[0], t_program[1])
751
        if not prog[0]:
752
753
            print("download_raw_images_by_program_via_png: Error! program not found, cannot download the data")
            return False,[0],[0]
754
        else:
755
756
            starttime=prog[1]['trigger']['1'][0]
            stoptime=prog[1]['trigger']['6'][0]
757
            OP=IR_tools.get_OP_by_time(time_ns=starttime)
758
            if version==0:
759
                version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM",t_from=starttime,t_to=stoptime)
760
            larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
761
762
763
            return download_images_by_time_via_png(larchivepath,starttime,stoptime,
                                                   time_window=time_window,threads=threads,
                                                   version=version,verbose=verbose-1)
764
            
765
    def download_images_by_time_via_png(larchivepath,starttime,stoptime,time_window=0,
766
                                        threads=1,version=0,verbose=0,framerate='max'):
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
        """
        """
        success=True            
        if type(time_window)==list:
            tstart=time_window[0]
            tstop=time_window[1]
            if tstop<tstart:
                raise Exception("download_images_by_time_via_png: endtime before starttime")
            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9+tstart)
            stdate=stdate.isoformat()
            enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+tstop)  
            enddate=enddate.isoformat()
        else:
            stdate=datetime.datetime.utcfromtimestamp((starttime-100)/1e9)
            stdate=stdate.isoformat()
            if time_window==0:
                enddate=datetime.datetime.utcfromtimestamp(stoptime/1e9)        
784
785
                enddate=enddate.isoformat()
            else:
786
787
788
789
790
791
                enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_window)  
                enddate=enddate.isoformat()
            #"2017-11-15 08:00:00"
        try:
            times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
        except Exception as E:
792
            print('download_images_by_time_via_png: Error loading times from ',larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))
793
            raise Warning(E)
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
            return False,[0],[0]  
            
        if framerate!='max': 
            if verbose>5:
                print("download_images_by_time_via_png: downsampling active; number of original frames:",len(times))
            if type(framerate)==float or type(framerate)==int:
                realrate=1e9/((times[0]-times[-1])[0]/len(times))
                if framerate<realrate:
                    steps=int(round(realrate/framerate,0))
                    dummy=[]
                    for i in range(0,len(times),steps):
                        dummy.append(times[i])
                    times=dummy
            if verbose>5:
                print("download_images_by_time_via_png: downsampling active; number of downsampeld frames:",len(times),steps)
809
810
811
812
813
814
815
816
817
818
        time=[]
        images=[]
        lnt=len(times)
        if threads==1:
            for i in range(lnt):
                ele=times[lnt-1-i]
                imag=download_last_raw_image_by_time(larchivepath,ele[0]-10,ele[0]+10)
                if imag[0]:
                    time.append(ele[0])
                    images.append(np.array(imag[1],dtype=np.uint16))
Holger Niemann's avatar
Holger Niemann committed
819
                else:
820
821
822
823
824
825
826
827
828
829
830
831
832
833
                    success=False
            return success,np.array(time),images
        else:
            tim=[]
            for i in range(lnt):
                tim.append(times[lnt-1-i][0])
            intervalls=[]
            intervalSize=int(lnt/threads)
            for i in range(threads):
                intervalls.append(int(i*intervalSize))
            intervalls.append(lnt)
            jobs = []
            resultdict = []
            for i in range(threads):
834
                if verbose>0:
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
                    print("download_images_by_time_via_png: Start Thread ",i+1)                
                TH=download_images_thread(i,larchivepath,tim[intervalls[i]:intervalls[i+1]],resultdict,version)
                jobs.append(TH)
                TH.start()
            for p in jobs:
                p.join()
            if verbose>0:
                print("download_images_by_time_via_png: all threads are done")
            order=[]
            for ele in resultdict:
                order.append(ele[0])
                if len(np.where(np.asarray(ele[1])==False)[0])>0:
                    success=False                
            times=[]#np.array(resultdict[order.index(0)][2])
            images=[]#np.array(resultdict[order.index(0)][3])
            for i in range(threads):
                images=images+resultdict[order.index(i)][3]
                times=times+resultdict[order.index(i)][2]
                resultdict[order.index(i)]=[]
            del resultdict
            return success,np.array(times),images
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
      
    class download_images_thread(threading.Thread):
        def __init__(self, threadID, larchivepath,times,resultlist,version=0):
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            self.resultlist.append([self.threadID,successes,time,images])
878
    
879
    class convert_raw_to_temp_thread(threading.Thread):
880
881
882
        def __init__(self,larchivepath,times,resultlist,threadID,version=0,background=0,
                     LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,
                     give_ERROR=False,FOV=0,badpixels=[],verbose=0):
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
899
            self.verbose=verbose
900
901
902
903
904
905
906
907
908
909
910
911
912
        def run(self):
            images=[]
            time=[]
            successes=[]        
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
            if self.give_ERROR:
913
                sucess,images,error_images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR,verbose=self.verbose-1)
914
            else:
915
                success,images=apply_calib_on_raw(images,self.background,self.LUT,self.refT,self.gain,self.offset,self.gain_error,self.offset_error,False,self.give_ERROR,verbose=self.verbose-1)
916
917
918
            
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
919
920
921
            if self.verbose>0:
                print(datetime.datetime.now(),"convert_raw_to_temp_thread: correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
922
            self.resultlist.append([self.threadID,successes,time,images])
923
    
924
    class NUC_raw_thread(threading.Thread):
925
926
        def __init__ (self,larchivepath,times,resultlist,threadID,version=0,background=0,
                      LUT=[[],[],[]],refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,
927
                      give_ERROR=False,FOV=0,badpixels=[],verbose=0):
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
            threading.Thread.__init__(self)
            self.threadID = threadID
            self.times = times
            self.archivepath = larchivepath
            self.resultlist = resultlist
            self.version = version   
            self.gain=gain
            self.offset=offset
            self.background=background
            self.LUT=LUT
            self.refT=refT
            self.gain_error=gain_error
            self.offset_error=offset_error
            self.give_ERROR=give_ERROR
            self.FOV=FOV
            self.badpixels=badpixels
944
            self.verbose=verbose
945
946
947
948
949
950
951
952
953
954
955
956
        def run(self):
            images=[]
            time=[]
            successes=[]
            for i in self.times:
                imag=download_last_raw_image_by_time(self.archivepath,i-10,i+10,version=self.version)
                if imag[0]:
                    images.append(np.array(imag[1],dtype=np.uint16))
                    time.append(i)
                    successes.append(True)
                else:
                    successes.append(False)
957
958
            if self.verbose>0:
                print(datetime.datetime.now(),"NUC_raw_thread: applying NUC")
959
960
            if self.give_ERROR:
                images,error_images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
961
            else:
962
963
964
                images=apply_NUC(images,self.gain,self.offset,self.gain_error,self.offset_error,self.give_ERROR)
            for i in range(len(images)):
                images[i]=(images[i]*self.FOV).astype(np.float32)
Peter Drewelow's avatar
Peter Drewelow committed
965
                
966
967
968
            if self.verbose>0:
                print(datetime.datetime.now(),"NUC_raw_thread: correcting bad pixels")
            images=IR_tools.correct_images(images,self.badpixels,verbose=self.verbose-1)
969
            self.resultlist.append([self.threadID,successes,time,images]) 
970

971
def download_raw_FLIR_images_via_png(t1,t6,time_window=0,version=0,threads=1,verbose=0,Test=True):
972
973
    """
    """
974
975
976
977
    if Test:
        larchivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+"W7X/QSR07_FLIR/AEK51_raw"
    else:
        larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
978
979
980
    if type(time_window)==list:
        tstart=time_window[0]
        tstop=time_window[1]
981
        if tstop<tstart:
982
            raise Exception("download_raw_FLIR_images_via_png: endtime before starttime")
983
984
985
986
987
<