Merge branch 'Holgers' into 'master'

Holgers

See merge request !31

downloadversionIRdata.py

@@ -4,7 +4,7 @@ Created on Wed Oct 25 15:51:46 2017
 updated on Tue Aug 21 10:20:00 2018
 last update on Fr Nov 23 15:37:00 2018
 
-Version: 3.2.1
+Version: 3.2.2
 (Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
 @author: holn
 """
@@ -45,6 +45,7 @@ except:
 
 
 def get_latest_version(stream,project="W7X",Test=False,t_from=None,t_to=None,program=None):
+    
      """Find out latest version of given stream
      :param stream   url stream of interest
      :return int of version number of None if non-versioned or non-existing stream
@@ -75,6 +76,10 @@ def get_latest_version(stream,project="W7X",Test=False,t_from=None,t_to=None,pro
      except urllib.error.HTTPError as ex:
          msg = ex.read()
          raise RuntimeError(msg)
+     except Exception as E:
+         raise RuntimeError(E)
+     else:
+         response.close()
      # detect unversioned or non-existing stream
      if d["versionInfo"] == []:
          return None
@@ -113,34 +118,16 @@ def read_program(timestamp_start,timestamp_end=0,tol=60):
         print('read_program: Error opening URL')
         print(e)
         return False,0
+    except Exception as e:
+        print('read_program: Error opening URL',e)
+        return False,0
     else:
         prog_string=prog_raw.decode(encoding='UTF-8')
         prog_list = json.loads(prog_string)
         pl=prog_list['programs'][0]
         return True, pl
 
-def read_restdb_old(request_url):
-    """
-    Reads JSON data from W7-X REST API
-    Returns: 
-        valid: access ok
-        t: numpy-array of time
-        signal: numpy-array of requested signals
-    by H. Thomsen
-    """
-    try:
-        res = urllib.request.urlopen(request_url)
-    except urllib.error.URLError as e:
-        print('read_restdb_old: Error ', e)
-        return False, 0, -1
-    else:
-        signal_raw=res.read()
-        res.close()
-        signal_string=signal_raw.decode(encoding='UTF-8')
-        signal_list = json.loads(signal_string)
-        signal0=np.array(signal_list['values'])
-        t=np.array(signal_list['dimensions'])
-        return True, t, signal0
+
 
 def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, verbose=0):
     """
@@ -236,6 +223,9 @@ def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
     except urllib.error.URLError as e:
         print('download_NUC_by_times: Error! ', e)
         goon=False
+    except Exception as e:
+        print('download_NUC_by_times: Error! ', e)
+        goon=False
     if goon:        
         n=0
         nuctimes=[0]
@@ -447,6 +437,9 @@ def download_hot_cold_reference_by_times(port,exposure,starttime=150390720000000
     except urllib.error.URLError as e:
         print('download_hot_cold_reference_by_times: Error! ',e)
         goon=False
+    except Exception as e:
+        print('download_hot_cold_reference_by_times: Error! ',e)
+        goon=False
     if goon:        
         COLDtime=0
         HOTtime=0
@@ -506,6 +499,9 @@ def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=
     except urllib.error.URLError as e:
         print('download_background_by_times: Error! ',e)
         goon=False
+    except Exception as e:
+        print('download_background_by_times: Error! ',e)
+        goon=False
     if goon: 
         n=0
         backtimes=[0]
@@ -627,6 +623,8 @@ def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalS
                 success=True
             except urllib.error.URLError as e:
                 print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e))                
+            except Exception as e:
+                print('download_images_by_times: Error in sub-interval {0}! {1}'.format(i,e))   
         return success,time,allimages
     else:
         try:
@@ -643,6 +641,9 @@ def download_images_by_times(larchivepath,starttime,stoptime,version=0,intervalS
         except urllib.error.URLError as e:
             print('download_images_by_times: Error! ',e)
             return False, 0,-1
+        except Exception as e:
+            print('download_images_by_times: Error! ',e)
+            return False, 0,-1
 
 if fastDL:
     def download_raw_images_by_program_via_png(port,program,time_window=0,version=0,threads=1,verbose=0):
@@ -1110,6 +1111,9 @@ def download_last_raw_image_by_time(larchivepath,starttime,stoptime,version=0):
     except urllib.error.URLError as e:
         print('download_last_raw_image_by_time: Error querrying {0}\n{1}'.format(url,e))
         return False, [-1]
+    except Exception as e:
+        print('download_last_raw_image_by_time: Error querrying {0}\n{1}'.format(url,e))
+        return False, [-1]
     
 def download_raw_parlog_by_program(port,program,version=0):
     """
@@ -1141,6 +1145,9 @@ def download_raw_parlog_by_times(port,starttime,stoptime,version=0):
     except urllib.error.URLError as e:
         print('download_raw_parlog_by_times: Error! ',e)
         goon=False
+    except Exception as E:
+        print('download_raw_parlog_by_times: Error! ',E)
+        goon=False
     if goon: 
         return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']
     
@@ -1175,6 +1182,9 @@ def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0):
     except urllib.error.URLError as e:
         print('get_INFRATEC_filter_by_times: Error! ',e)
         goon=False
+    except Exception as E:
+        print('get_INFRATEC_filter_by_times: Error! ',E)
+        goon=False
     if goon: 
         return True, signal_list['dimensions'],signal_list['values'][0]['meta-data']['filter']  
     else:
@@ -1787,6 +1797,8 @@ def get_temp_from_raw_by_program(portnr,program,time_window=0,emi=0.8,T_version=
                             
                             for i in range(len(frames)):
                                 frames[i]=(frames[i]*FOV).astype(np.float32)
+                            
+#                            if portnr!=50:
                             if verbose>0:
                                 print(datetime.datetime.now(),"get_temp_from_raw_by_program: correcting bad pixels")
                             frames=IR_tools.correct_images(frames,badpixels)
@@ -2098,7 +2110,7 @@ def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_erro
 #        if fullbackground:
         #sbackground=np.zeros(np.shape(images[0]),dtype=np.float32)+background
         for i in range(len(images)):
-            images[i]=np.clip((images[i]-background),a_min=0,a_max=None).astype(np.uint16)
+            images[i]=np.clip((images[i].astype(np.int32)-background),a_min=0,a_max=None).astype(np.uint16)
 #        else:
 #            for i in range(len(images)):
 #                images[i]=images[i]-background
@@ -2628,7 +2640,7 @@ def download_heatflux_by_times(port,tstart,tend,time_window=0,threads=1,testmode
                 if verbose>0:
                     print(now,"download_heatflux_by_times: heat flux data is not available, creating request")          
                 if request:
-                    f=open(heatflux_requestlist_path+str(now.year)+str(now.month)+"_q_requests.txt",'a')
+                    f=open(heatflux_requestlist_path+str(now.year)+str(now.month-1)+"_q_requests.txt",'a')
                     try:
                         programid=AKF_2.get_program_id(tstart)
                     except Exception as E:
@@ -2711,6 +2723,9 @@ def download_heatflux_scene_model_reference(port,timepoint=None,program=None,ver
     except urllib.error.URLError as e:
         print('download_heatflux_scene_model_reference: Error! ',e,larchivepath_par+"/_signal.json?from="+str(timepoint)+"&upto="+str(timepoint))
         goon=False
+    except Exception as E:
+        print('download_heatflux_scene_model_reference: Error! ',E)
+        goon=False
     if goon:
         timepoint=signal_list['values'][0]  
         exist,time,frames=download_images_by_times(larchivepath,starttime=timepoint-10,stoptime=int(timepoint+1e8),version=version,verbose=verbose-1)        
@@ -2878,7 +2893,7 @@ def get_heatflux_profile(port,profile,finger=None,timepoint=1,program=None,tstar
                 return False,0,0,0
 
 def extract_heatflux_profile_from_DL(time,images,profile,finger=None,time_window=1,
-                                     inputchecked=False,verbose=0):
+                                     inputchecked=False,verbose=0,testmode=False):
     """
     return exist,time,s in m, heat flux in W/m2
     time_point default (None) returns entire shot.
@@ -2922,7 +2937,7 @@ def extract_heatflux_profile_from_DL(time,images,profile,finger=None,time_window
         else:
             fingerID=finger*100+profile
         ### okay finderID is fine, data is given, lets get the mapping
-        exist,mapping=download_heatflux_mapping_reference()
+        exist,mapping=download_heatflux_mapping_reference(testmode=testmode)
         if exist:
             Fingermap=np.asarray(mapping['Finger_ID'][0],dtype=np.int)            
             locs=np.where(Fingermap==fingerID)
@@ -3177,10 +3192,36 @@ def download_scene_model(port,program=None,timepoint=None,version=0,verbose=0):
         
 #%% general download functions
 """
-the following functions are copied from W7xrest.read_restdb to remove this dependency
+the following functions are copied from W7xrest.read_restdb to remove this dependency and are edit to fit into the other functions
 @author: thomsen
 """
 
+def read_restdb_old(request_url):
+    """
+    Reads JSON data from W7-X REST API
+    Returns: 
+        valid: access ok
+        t: numpy-array of time
+        signal: numpy-array of requested signals
+    by H. Thomsen
+    """
+    try:
+        res = urllib.request.urlopen(request_url)
+    except urllib.error.URLError as e:
+        print('read_restdb_old: Error ', e)
+        return False, 0, -1
+    except Exception as e:
+        print('read_restdb_old: Error ', e)
+        return False, 0, -1
+    else:
+        signal_raw=res.read()
+        res.close()
+        signal_string=signal_raw.decode(encoding='UTF-8')
+        signal_list = json.loads(signal_string)
+        signal0=np.array(signal_list['values'])
+        t=np.array(signal_list['dimensions'])
+        return True, t, signal0
+
 def read_restdb(request_url):
     """
     Reads JSON data from W7-X REST API
@@ -3204,6 +3245,9 @@ def read_restdb(request_url):
             #print('The server couldn\'t fulfill the request.')
             #print('Error code: ', e.code)
         return False, [0], [-1]
+    except Exception as E:
+        print('read_restdb: Error! ',E)
+        return False, [0], [-1]
     else:
 #        res = urllib.request.urlopen(request_url)
 #        signal_raw=res.read()
@@ -3262,7 +3306,10 @@ def get_program_from_PID(instring):
         prog_list = json.loads(res.read().decode('utf-8'))
         res.close()
     except urllib.error.URLError as e:
-        print('get_program_from_PID: Error opening URL')
+        print('get_program_from_PID: Error opening URL',e)
+        return False,0
+    except Exception as E:
+        print('get_program_from_PID: ERROR appeared',E)
         return False,0
     else:
         pl=prog_list['programs']
@@ -3288,24 +3335,30 @@ if __name__=='__main__':
     print("local function calling") 
     
 #%% temperature download and plotting example                  
-#    port=50#"AEF50"#"AEF51"
-#    prog="20180920.042"#"20171109.021"#"20181010.036"
-#    status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_window=[0,2],emi=0.82,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=50,framerate=1)
+    port=50#"AEF50"#"AEF51"
+    prog='20170927.020'#"20181011.036"#"20181016.037"#"20180920.042"#"20171109.021"#"20181010.036"
+    status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_window=[0,4],emi=0.80,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
+#    status2,time2,images2=download_raw_images_by_program(port,prog,time_window=0.02,verbose=5)
+#    bla=get_calib_data(50,program=prog,verbose=5)
 #    success,t,s,profile=extract_temperature_profile_from_DL(port,np.asarray(time-time[0])/1e9,images,profile="TM3h_5_5",verbose=10,reference_time=time[0])
-#    if status:
-#        plt.figure()
-#        plt.imshow(images[0],vmin=330,vmax=1000,cmap=exJet)
-#        cb=plt.colorbar()
-#        cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20)
-#        cb.ax.tick_params(labelsize=20)
+    if status:
+        plt.figure()
+        plt.imshow(images[0],vmin=330,vmax=1000,cmap=exJet)
+        cb=plt.colorbar()
+        cb.set_label("temperature in K",rotation=270,labelpad=20,fontsize=20)
+        cb.ax.tick_params(labelsize=20)
 #%% heatflux test
 #    port=10
-#    prog="20171114.039"    
-#    status,time,images=download_heatflux_by_program(port,prog,timewindow=0.5,threads=1,version=1,verbose=3,givealpha=False,request=True)
-#    if status:
-#        print(prog,port,"OK")
-#    else:
-#        print(prog,port,"missing")
+#    prog="20180904.015" 
+#    
+#    for prog in ["20180823.016","20180918.045","20181016.037","20181009.024","20181016.016"]:#"20171207.024","20180814.024","20181016.016","20181010.036"]:
+#        for port in [10,11,20,21,30,31,40,41,50,51]:
+#            status,time,images=download_heatflux_by_program(port,prog,time_window=4,threads=4,version=2,verbose=3,givealpha=False,request=True)            
+##            status,time,load,error=download_divertor_load(port,targetmodule=['all','tm1h'],program=prog,verbose=5,version=3)
+#            if status:
+#                print(prog,port,"OK")
+#            else:
+#                print(prog,port,"missing")
 #        plt.figure(),plt.imshow(images[0],vmin=0)
 #        plt.figure(),plt.imshow(images[50],vmin=0)
 #    status,mapping=download_heatflux_mapping_reference(verbose=4)

plot_IR_data.py

@@ -15,7 +15,7 @@ if __name__=='__main__':
     
     #%% loading data
     print(datetime.datetime.now(),"start")
-    status,time,images,valid=downIR.get_temp_from_raw_by_program(10,"20180801.017",time_s=2.5,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,verbose=5)
+    status,time,images,valid=downIR.get_temp_from_raw_by_program(10,"20180801.017",time_window=2.5,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,verbose=5)
     print('fertig')
     
     #%% plotting data

plot_heatflux_example.py

@@ -5,7 +5,7 @@ V3.2.0
 @author: holn
 """
 import numpy as np
-import IR_data_access.downloadversionIRdata as IR
+import downloadversionIRdata as IR
 import matplotlib.pyplot as plt
 import datetime
 
@@ -16,12 +16,12 @@ if __name__=='__main__':
     
     #%% loading data
     print(datetime.datetime.now(),"start")
-    status,times,images=IR.download_heatflux_by_program(port,program,time_window=0.9,version=1,threads=1,verbose=5)
+    status,times,images=IR.download_heatflux_by_program(port,program,time_window=6.5,version=2,threads=1,verbose=5,testmode=False)
     print('done')
     
     #%% plotting data
     if status:
-        status2,time,s,q=IR.extract_heatflux_profile_from_DL((np.asarray(times)-times[0])/1e9,images,profile="TM3h_5_5",time_window=0.5,verbose=5)        
+        status2,time,s,q=IR.extract_heatflux_profile_from_DL((np.asarray(times)-times[0])/1e9,images,profile="TM3h_5_5",time_window=None,verbose=5,testmode=False)        
         plt.figure()
         plt.imshow(images[-1]/1e6)
         plt.clim([0,4])