Merge branch 'Holgers' into 'master'

Update 3.2.5

See merge request !39

IR_config_constants.py

 # -*- coding: utf-8 -*-
 """
 Created on Tue Jun  5 09:26:06 2018
-Version: 3.0.0
+Version: 3.2.5
 @author: holn
 constants and config parameters
 """
 #%% Paths
 calibpath="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Calibrations\\"
 IRCamColdframes_fittingpath=calibpath+"IRCamColdframes_fitting\\"
-IRCamRefImagespath = calibpath+'IRCamReferenceImages\\'
+#IRCamRefImagespath = calibpath+'IRCamReferenceImages\\'
 IRCAMBadPixels_path = calibpath + 'IRCAMBadPixels\\'
 heatflux_requestlist_path="\\\\x-drive\\Diagnostic-logbooks\\QRT-DivertorThermography\\ArchiveDB_heatflux_pipeline\\"
 parameter_file_path = ".\\data"

IR_image_tools.py

 # -*- coding: utf-8 -*-
 """
 Created on Wed May  9 14:56:32 2018
-Version: 3.1.0
+Version: 3.2.5
 @author: Holger Niemann, Peter Drewelow, Yu Gao
 
 mainly to clean up the downloadversionIRdata code
@@ -15,11 +15,11 @@ Tools for:
 import numpy as np
 import matplotlib.pyplot as plt
 import matplotlib.patches as patches
-from IR_config_constants import portcamdict,IRCamRefImagespath,IRCAMBadPixels_path,parameter_file_path
-import h5py
+from IR_config_constants import portcamdict,IRCAMBadPixels_path,parameter_file_path
 import os
-import glob
 import datetime
+#import h5py
+#import glob
 
 # set working directory to local directory of script
 os.chdir(os.path.dirname(__file__))
@@ -478,29 +478,31 @@ def calculate_gain_offset_image(cold_image,hot_image=None,reference_cold=None,re
     
 #%% functions from Yu Gao
 """ functions by Yu Gao"""
-    
-def load_ref_images(port, exposuretime, verbose=0):
-    '''
-    load the reference cold and hot frame during calibration from local files.
-    @port: e.g. 'AEF10'
-    @exposuretime: int number.
-    '''
-    cameraname = portcamdict['OP1.2a'][port]
-    foldername = cameraname.split('_')[0] + '_' + cameraname.split('_')[2]
-    scanpath = os.path.join(IRCamRefImagespath, foldername)
-    coldref, hotref = [], []
-    for filename in glob.iglob(scanpath + '\*' + str(int(exposuretime)) + 'us.h5', recursive=True):
-        if 'hot' in filename:
-            if verbose>0:
-                print('load_ref_images: read from ',filename)
-            with h5py.File(filename, 'r') as h5in:
-                hotref = h5in[os.path.basename(filename)].value
-        elif 'cold' in filename:
-            if verbose>0:
-                print('load_ref_images: read from ',filename)
-            with h5py.File(filename, 'r') as h5in:
-                coldref = h5in[os.path.basename(filename)].value
-    return coldref, hotref
+#outdated by download_hot_cold_reference in downloadversionIRdata. removed to remove dependency on data on the E4-drive    
+#==============================================================================
+# def load_ref_images(port, exposuretime, verbose=0):
+#     '''
+#     load the reference cold and hot frame during calibration from local files.
+#     @port: e.g. 'AEF10'
+#     @exposuretime: int number.
+#     '''
+#     cameraname = portcamdict['OP1.2a'][port]
+#     foldername = cameraname.split('_')[0] + '_' + cameraname.split('_')[2]
+#     scanpath = os.path.join(IRCamRefImagespath, foldername)
+#     coldref, hotref = [], []
+#     for filename in glob.iglob(scanpath + '\*' + str(int(exposuretime)) + 'us.h5', recursive=True):
+#         if 'hot' in filename:
+#             if verbose>0:
+#                 print('load_ref_images: read from ',filename)
+#             with h5py.File(filename, 'r') as h5in:
+#                 hotref = h5in[os.path.basename(filename)].value
+#         elif 'cold' in filename:
+#             if verbose>0:
+#                 print('load_ref_images: read from ',filename)
+#             with h5py.File(filename, 'r') as h5in:
+#                 coldref = h5in[os.path.basename(filename)].value
+#     return coldref, hotref
+#==============================================================================
 
 def reconstruct_coldframe (exposuretime, sT, a, bnew, coldref):
     cirebuild = a * sT + bnew * exposuretime + coldref

downloadversionIRdata.py

@@ -4,7 +4,7 @@ Created on Wed Oct 25 15:51:46 2017
 updated on Tue Aug 21 10:20:00 2018
 last update on Fr Nov 23 15:37:00 2018
 
-Version: 3.2.4
+Version: 3.2.5
 (Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
 @author: holn
 """
@@ -486,46 +486,63 @@ def download_background_by_program(port,program,exposure,version=0):
         print("download_background_by_program: Error! cannot find the program {0}".format(program))
         return False,0,0
     
-def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
+def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0,verbose=0):
     OP=IR_tools.get_OP_by_time(time_ns=starttime)
-    stream=portpathdict[OP]["AEF"+str(port)]+"background_"
-    larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
-    if version==0:
-        version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
-    try:
-        res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
-        signal_list = json.loads(res.read().decode('utf-8'))
-        res.close()
-        goon=True
-    except urllib.error.URLError as e:
-        print('download_background_by_times: Error! ',e)
-        goon=False
-    except Exception as e:
-        print('download_background_by_times: Error! ',e)
-        goon=False
-    if goon: 
-        n=0
-        backtimes=[0]
-        for backpar in signal_list['values']:
-            if backpar['structure']['Texp']==exposure:
-                if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
-                    if backpar['structure']['filter']==camera_filter:
+    if OP=="OP1.2a":
+        stream=portpathdict[OP]["AEF"+str(port)]+"background_"
+        larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"background_"
+        if version==0:
+            version=get_latest_version(stream+"DATASTREAM",t_from=starttime)
+        try:
+            res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
+            signal_list = json.loads(res.read().decode('utf-8'))
+            res.close()
+            goon=True
+        except urllib.error.URLError as e:
+            print('download_background_by_times: Error! ',e)
+            goon=False
+        except Exception as e:
+            print('download_background_by_times: Error! ',e)
+            goon=False
+        if goon: 
+            n=0
+            backtimes=[0]
+            for backpar in signal_list['values']:
+                if backpar['structure']['Texp']==exposure:
+                    if port==50:# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
+                        if backpar['structure']['filter']==camera_filter:
+                            backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
+                    else:
                         backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
+                n+=1
+            if backtimes[0]!=0:
+                backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
+                if backdat[0]:                                    
+                    return backdat#[True,backdat[2]]
                 else:
-                    backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]                             
-            n+=1
-        if backtimes[0]!=0:
-            backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
-            if backdat[0]:                                    
-                return backdat#[True,backdat[2]]
+                    print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
+                    return False,0,0
             else:
                 print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
                 return False,0,0
         else:
-            print("download_background_by_times: Error! background image for requested exposure time(, filter) not found")
             return False,0,0
-    else:
-        return False,0,0
+    elif OP=="OP1.2b":
+        NUC_DL=get_NUC_by_times(port=port,starttime=starttime,endtime=stoptime,t_exp=exposure,version=version,verbose=verbose-1)#download_NUC_by_times(port,t0,t1,t_exp,version)
+        if NUC_DL[0]==False:
+            raise Warning("download_background_by_times: NUC was not found")
+            return False,0,0
+        else:
+            # extract NUC elements
+            gain = []
+            offset = []
+            gain=np.array(NUC_DL[1][0])
+            offset=np.array(NUC_DL[1][1])
+            # check quality of NUC elements
+            gain[np.isnan(gain)]=0
+            offset[np.isnan(offset)]=0
+            offset[offset<-1e100]=0
+            return get_NUCed_background_by_times(port=port,t0=starttime,t1=stoptime,t_exp=exposure,cfilter=camera_filter,gain=gain,offset=offset,version=version,verbose=verbose-1)
 
 def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False,verbose=0):
     """