Commit d9950a42 authored by Peter Drewelow's avatar Peter Drewelow
Browse files

merging master into peter

parents c3fdfc45 96355a14
*.sln
*.opendb
*.db
*.pdb
*.pyc
.vs
/__pycache__
/.hg
<<<<<<< HEAD
27/28/29.11.2018: update to V3
- add file upload_heatflux.py
- add plot_heatflux_example.py
update downloadversionIRdata:
- add functions download_heatflux_by_program,download_heatflux_by_times, download_heatflux_profile, extract_heatflux_profile_from_DL
- rename download_raw_images_by_time_via_png into download_images_by_time_via_png
update uploadversionIRdata:
- add function write_H5_images, a new less ram intense version of write_H5 function for better image upload (specially heatflux)
26.09.2018:
update downloadversionIRdata:
- include FLIR into the downloadscript, Case1 (V2.8.0)
25.09.2018:
update downloadversionIRdata:
- improved performance (V2.7.1), implement switch between test and ArchiveDB (V2.7.0)
13.09.2018:
update downloadversionIRdata:
- bugfixes, verbose level passed now throw, V2.6.1
12.09.2018:
update downloadversionIRdata:
- ToDo list added
- file added to create temperature HDF5 files
21.08.2018:
update downloadversionIRdata:
- remove dependency on readrestdb V2.6.0
- include version number, now 2.5.3 into the files
- bug fixes for downloadversion IRdata
- improvement of exception handling, replace some exception with warnings and improve error messages
=======
01.08.2017:
changes in downloadversionIRdata:
- OP1.2b version now online for upload and download of the IRdata
>>>>>>> 868a427421003a9ced34cbbf3c17eadf42312057
03.11.2017:
changes in downloadversionIRdata:
- reduce blocks to 1 second, now editable over parameter intervalSize
......@@ -7,4 +40,24 @@ changes in downloadversionIRdata:
bugfixes in downloadversionIRdata:
- fixed: stepwise download of data in blocks of 2 seconds (used the wrong indices)
- fixed: not find the infratecfilter
- fixed: wrong time intervall for TC divertor data
\ No newline at end of file
- fixed: wrong time intervall for TC divertor data
Versions:
V3.0.0: download and upload of heat flux data now also possible, the script creates a request+memory file for all heatflux request which point to a camera, program comibination which was not uploaded; in this way high requested heatflux data can be evaluated earlier if possible
V2.9.7: bug fixes in the download for OP1.2a data, background download was broken
V2.9.6: bug fixes for AEFXX, port and portnr was not used in the right way
V2.9.5: bug fixes for AEFXX, wrong dictionaries was used, use the correct defintion of the paths for the download.
V2.9.4: bug fixes for AEF50 download in OP1.2a (got trouble with the change to real Archive), fixes in the upload script to upload flir data and fix extra frames
V2.9.3: bugfixes for FLIR, ArchiveDB is now used as default as download
V2.9.2: bugfixes for AEF50 in the download
V2.9.1: bugfixes for AEF50
V2.9.0: changed multiprocessing to multithreading, better performance, less archive calls
V2.8.0: add FLIR camera into the downloadroutine, adapt the port interface, that it accept strings and numbers, numbers are automatically used for QRT
V2.7.1: prove the temperature calculation, less ram usage
V2.7.0: implement dependency on upload_config; switch between ArchiveDB and Test archive possible
V2.6.1: bugfixes with verbose level
V2.6.0: dependency on read_restdb removed
V2.5.3: version number included; improved exception handeling
V2.0.0: upgrade an changes to download OP1.2b data, changes of the method and what is stored
V1.1.0: download of thermocouples implemented
V1.0.0: download test of OP1.2a data
\ No newline at end of file
# -*- coding: utf-8 -*-
"""
Created on Mon Dec 4 13:28:31 2017
@author: holn
save V2 into new files
"""
if __name__=='__main__':
import h5py as h5
import downloadversionIRdata as IR_DL
import numpy as np
localpath=""
time_s=13
programlist = ["20171121.013"]
ports=[10]
for program in programlist:
for port in ports:
try:
exist,dimof,data1,valid=IR_DL.get_temp_from_raw_by_program(portnr=port,program=program,time_s=time_s,threads=4,verbose=5)
if exist:
print("data found")
stream="images"
data1=np.asarray(data1)
data1=data1.swapaxes(0,2)
data1=data1.swapaxes(0,1)
dtype = str(data1.dtype)
tmpfile = localpath+"AEF"+str(port)+"_"+program+"_temp_V2"
tmpfile += ".h5"
with h5.File(tmpfile, 'w') as f:
f.create_dataset('timestamps', data=dimof, dtype='uint64' )#,compression="gzip")
dset=f.create_dataset(stream, shape=np.shape(data1),dtype=dtype , chunks = (np.shape(data1)[0],np.shape(data1)[1],1))
data1=np.ascontiguousarray(data1)
dset.write_direct(data1)
del data1
except Exception as E:
print(E)
\ No newline at end of file
# -*- coding: utf-8 -*-
"""
Created on Tue Jun 5 09:26:06 2018
Version: 3.0.0
@author: holn
constants and config parameters
"""
#%% Paths
archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/W7X/"
calibpath="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Calibrations\\"
IRCamColdframes_fittingpath=calibpath+"IRCamColdframes_fitting\\"
IRCamRefImagespath = calibpath+'IRCamReferenceImages\\'
IRCAMBadPixels_path = calibpath + 'IRCAMBadPixels\\'
heatflux_requestlist_path="\\\\x-drive\\Diagnostic-logbooks\\QRT-DivertorThermography\\ArchiveDB_heatflux_pipeline\\"
try:
# import sys
path=str(repr(__file__)).split("IR_config_constants")[0].split("'")[1]+"upload_config"
cFile=open(path.replace('\\\\',"\\"))#+"upload_config")
for line in cFile:
if line[0:3]!="###":
if line.split("=")[0]=='archivedb':
if eval(line.split("=")[1].split("\n")[0]):
archivepath="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/"
else:
archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
cFile.close()
except Exception as E:
archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
print("config loading failed",E)
# raise Exception("Config file was not loaded properly",E)
#%% Dictionaries
project="W7X"
project_ana="W7XAnalysis"
portcamdict={
'OP1.2a':{
'AEF10': 'IRCam_Caleo768kL_0901',
......@@ -24,7 +42,45 @@ portcamdict={
'AEF40': 'IRCam_Caleo768kL_0701',
'AEF41': 'IRCam_Caleo768kL_0902',
'AEF50': 'Infratec_9312907',
'AEF51': 'IRCam_Caleo768kL_0905'}
'AEF51': 'IRCam_Caleo768kL_0905'},
'OP1.2b':{
'AEF10': 'IRCam_Caleo768kL_0901',
'AEF11': 'IRCam_Caleo768kL_0904',#is corrected from 0906, the cameras was swapped in the campaign in restect to the planned position
'AEF20': 'IRCam_Caleo768kL_0702',
'AEF21': 'IRCam_Caleo768kL_0906',#is corrected from 0904, the cameras was swapped in the campaign in restect to the planned position
'AEF30': 'IRCam_Caleo768kL_0907',
'AEF31': 'IRCam_Caleo768kL_0903',
'AEF40': 'IRCam_Caleo768kL_0701',
'AEF41': 'IRCam_Caleo768kL_0902',
'AEF50': 'Infratec_9312907',
'AEF51': 'IRCam_Caleo768kL_0905',
'AEK51': 'FLIR_SC8303_00037'}
}
portpathdict={
'OP1.2a':{
'AEF10': 'QRT_IRCAM/AEF10_',
'AEF11': 'QRT_IRCAM/AEF11_',
'AEF20': 'QRT_IRCAM/AEF20_',
'AEF21': 'QRT_IRCAM/AEF21_',
'AEF30': 'QRT_IRCAM/AEF30_',
'AEF31': 'QRT_IRCAM/AEF31_',
'AEF40': 'QRT_IRCAM/AEF40_',
'AEF41': 'QRT_IRCAM/AEF41_',
'AEF50': 'QRT_IRCAM/AEF50_',
'AEF51': 'QRT_IRCAM/AEF51_'},
'OP1.2b':{
'AEF10': 'QRT_IRCAM/AEF10_',
'AEF11': 'QRT_IRCAM/AEF11_',
'AEF20': 'QRT_IRCAM/AEF20_',
'AEF21': 'QRT_IRCAM/AEF21_',
'AEF30': 'QRT_IRCAM/AEF30_',
'AEF31': 'QRT_IRCAM/AEF31_',
'AEF40': 'QRT_IRCAM/AEF40_',
'AEF41': 'QRT_IRCAM/AEF41_',
'AEF50': 'QRT_IRCAM/AEF50_',
'AEF51': 'QRT_IRCAM/AEF51_',
'AEK51': 'QSR07_FLIR/AEK51_'}
}
camlimdict={
......@@ -371,4 +427,14 @@ TC_database={
12: {'Y': 3650.051, 'Z': -1027.995, 'Targetmodule': 'TMR_TM3h', 'active': True, 'X': -3760.329, 'Targetelement': 0},
13: {'Y': 3871.122, 'Z': -1136.99, 'Targetmodule': 'DR_TM3h', 'active': True, 'X': -3940.342, 'Targetelement': 0},
14: {'Y': 3545.175, 'Z': -740.545, 'Targetmodule': 'TMR_TM2v', 'active': True, 'X': -3446.075, 'Targetelement': 0},
15: {'Y': 3628.857, 'Z': -714.53, 'Targetmodule': 'DR_TM2v', 'active': True, 'X': -3425.679, 'Targetelement': 0}}}
\ No newline at end of file
15: {'Y': 3628.857, 'Z': -714.53, 'Targetmodule': 'DR_TM2v', 'active': True, 'X': -3425.679, 'Targetelement': 0}}}
from matplotlib import cm
c2dict={
'red' : ((0,0,0),(0.166, 0, 0), (0.33, 0, 0), (0.49, 0, 0), (0.66, 1, 1), (0.82, 1, 1), (1, 1, 1)),
'green': ((0,0,0),(0.166, 0, 0), (0.33, 1, 1), (0.49, 1, 1), (0.66, 0, 0), (0.82, 1, 1), (1, 1, 1)),
'blue' : ((0,0,0),(0.166, 1, 1), (0.33, 1, 1), (0.49, 0, 0), (0.66, 0, 0), (0.82, 0, 0), (1, 1, 1))
}
c3dict=c2dict.copy()
exJet=cm.colors.LinearSegmentedColormap('mod_Jet', c2dict, 1024)
c3dict['alpha']=((0.0,0.0,0.0),(0.33,0.99,0.99),(1.0,1.0,1.0))
exJet_trans=cm.colors.LinearSegmentedColormap('mod_Jet_trans', c3dict, 1024)
\ No newline at end of file
# -*- coding: utf-8 -*-
"""
Created on Wed May 9 14:56:32 2018
Version: 2.8.0
@author: Holger Niemann, Peter Drewelow, Yu Gao
mainly to clean up the downloadversionIRdata code
......@@ -20,7 +20,6 @@ from os.path import join, basename
import glob
import datetime
def get_OP_by_time(time_ns=None, shot_no=None, program_str=None):
'''Derives operation phase (OP) of W7-X based on either:
a nanosacond time stamp, a MDSplus style shot no. or a program ID.
......@@ -48,7 +47,7 @@ def get_OP_by_time(time_ns=None, shot_no=None, program_str=None):
if dateOP.year == 2017:
if dateOP.month>8 and dateOP.month<12:
return "OP1.2a"
elif dateOP.month==8 and dateOP.day>28:
elif dateOP.month==8 and dateOP.day>=28:
return "OP1.2a"
elif dateOP.month==12 and dateOP.day<8:
return "OP1.2a"
......@@ -62,8 +61,7 @@ def get_OP_by_time(time_ns=None, shot_no=None, program_str=None):
else:
return None
def bestimmtheitsmaß_general(data,fit):
def bestimmtheitsmass_general(data,fit):
R=0
if len(fit)==len(data):
mittel=np.sum(data)/len(data)
......@@ -88,6 +86,14 @@ def quad_abweich(data,fit):
print("Arrays must have same dimensions")
return R
def find_nearest(array,value):
#a=array
a = [x - value for x in array]
mini = np.min(np.abs(a))
try: idx= a.index(mini)
except: idx= a.index(-mini)
return idx#array[idx]
def check_coldframe(coldframe,references=None,threshold=0.5,plot_it=False):
'''
return true/false and the quality factor
......@@ -113,7 +119,7 @@ def check_coldframe(coldframe,references=None,threshold=0.5,plot_it=False):
for i_dat in range(len(datasets)):
dat=datasets[i_dat]
reference=references[i_dat]
bestimmtheit.append(bestimmtheitsmaß_general(dat,reference))
bestimmtheit.append(bestimmtheitsmass_general(dat,reference))
if plot_it:
plt.plot(dat,label='data')
plt.plot(reference,label='reference')
......@@ -147,7 +153,7 @@ def check_backgroundframe(backgroundframe,threshold=50):
valid=False
return valid,np.mean(dataset)
def read_bad_pixels_from_file(port, shot_no=None, program=None):
def read_bad_pixels_from_file(port, shot_no=None, program=None,time_ns=None):
'''Reads bad pixels stored in *.bpx file on E4 server.
Requires one of the optional arguments shot_no or program.
IN
......@@ -163,13 +169,18 @@ def read_bad_pixels_from_file(port, shot_no=None, program=None):
OP = get_OP_by_time(shot_no=shot_no)
elif program is not None:
OP = get_OP_by_time(program_str=program)
elif time_ns is not None:
OP = get_OP_by_time(time_ns=time_ns)
else:
raise Exception('read_bad_pixels_from_file: ERROR! Need either shot no. or program string.')
port_name = 'AEF{0}'.format(port)
bad_pixel_file = 'badpixel_{0}.bpx'.format(portcamdict[OP][port_name][6:])
data = np.genfromtxt(IRCAMBadPixels_path+bad_pixel_file, dtype=int)
bad_pixle_list = list(zip(data[:,1], data[:,0]))
try:
data = np.genfromtxt(IRCAMBadPixels_path+bad_pixel_file, dtype=int)
bad_pixle_list = list(zip(data[:,1], data[:,0]))
except:
bad_pixle_list=[]
return bad_pixle_list
......@@ -211,14 +222,12 @@ def find_outlier_pixels(frame,tolerance=3,worry_about_edges=True,plot_it=False):
return bad_pixels
def correct_images(images,badpixels):
print('correct_images: New routine restore_bad_pixels() is used and can be called directly. Check out "help(restore_bad_pixels)"')
if type(badpixels)!=int:
if type(images) == list:
# return corrected images also as list of 2D arrays
images = restore_bad_pixels(images, np.invert(badpixels==1))
images = list(images)
for i in range(len(images)):
images[i]=restore_bad_pixels(images[i], np.invert(badpixels==1))
else:
# keep shape
images = restore_bad_pixels(images, np.invert(badpixels==1))
......@@ -281,7 +290,10 @@ def restore_bad_pixels(frames, bad_pixel, by_list=True, check_neighbours=True, p
blist = bad_pixel
bmask = np.ones([n_rows, n_cols],dtype=bool)
for pix in blist:
bmask[pix] = False
try:
bmask[pix] = False
except Exception as E:
Warning(E)
bmask = np.invert(bmask)
else:
if np.shape(bad_pixel)[0] == n_rows and np.shape(bad_pixel)[1] == n_cols:
......@@ -397,13 +409,13 @@ def restore_bad_pixels(frames, bad_pixel, by_list=True, check_neighbours=True, p
def generate_new_hot_image(cold,reference_cold,reference_hot):
if cold==None or reference_cold==None or reference_hot==None:
if cold is None or reference_cold is None or reference_hot is None:
raise Exception("Cannot Calculate new Hot image, if images are missing!")
else:
return reference_hot+(cold-reference_cold)
def calculate_gain_offset_image_pix(cold_image,hot_image=None,reference_cold=None,reference_hot=None,bose=1):
if hot_image==None:
if hot_image is None:
hot_image=generate_new_hot_image(cold_image,reference_cold,reference_hot)
if bose>0:
print("calculate gain and offset")
......@@ -415,24 +427,27 @@ def calculate_gain_offset_image_pix(cold_image,hot_image=None,reference_cold=Non
Offset_rel = ( Off_h_rel + Off_c_rel ) /2
return Gain_rel,Offset_rel
def calculate_gain_offset_image(cold_image,hot_image=None,reference_cold=None,reference_hot=None,bose=1):
if hot_image==None:
def calculate_gain_offset_image(cold_image,hot_image=None,reference_cold=None,reference_hot=None,verbose=0):
if hot_image is None:
hot_image=generate_new_hot_image(cold_image,reference_cold,reference_hot)
if bose>0:
if verbose>0:
print("calculate gain and offset")
# Sh_ref = hot_image[ ( np.int( np.shape(hot_image)[0] /2 ) ) ][np.int( (np.shape(hot_image)[1] /2 ) ) ]
# Sc_ref = cold_image[ ( np.int( (np.shape(cold_image)[0]) /2 ) ) ][( np.int( (np.shape(cold_image)[1]) /2 ) ) ]
print(hot_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])
print(cold_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])
# print(hot_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])
# print(cold_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])
Sh_ref = np.mean( hot_image[( np.int( np.shape(hot_image)[0]/2) )-2: (np.int( np.shape(hot_image)[0]/2))+3,np.int((np.shape(hot_image)[1]/2))-2:np.int((np.shape(hot_image)[1]/2))+3 ])
Sc_ref = np.mean(cold_image[( np.int( np.shape(cold_image)[0]/2) )-2: (np.int( np.shape(cold_image)[0]/2))+3,np.int((np.shape(cold_image)[1]/2))-2:np.int((np.shape(cold_image)[1]/2))+3 ])
Gain_rel = ( Sh_ref - Sc_ref ) / ( hot_image - cold_image)
difference_image=hot_image - cold_image
indexlist=np.where(difference_image==0)
difference_image[indexlist]=0.001
Gain_rel = ( Sh_ref - Sc_ref ) / ( difference_image)
Gain_rel[indexlist]=0
Off_h_rel = Sh_ref - hot_image*Gain_rel
Off_c_rel = Sc_ref - cold_image*Gain_rel
Offset_rel = ( Off_h_rel + Off_c_rel ) /2
return Gain_rel,Offset_rel
#%% functions from Yu Gao
""" functions by Yu Gao"""
......@@ -460,4 +475,20 @@ def load_ref_images(port, exposuretime):
def reconstruct_coldframe (exposuretime, sT, a, bnew, coldref):
cirebuild = a * sT + bnew * exposuretime + coldref
return cirebuild
\ No newline at end of file
return cirebuild
#%% other functions
def check_dublicates(array):
a = array
import collections
return [item for item, count in collections.Counter(a).items() if count > 1]
def check_dublicates_2(array):
seen = set()
uniq = []
for x in array:
if x not in seen:
uniq.append(x)
seen.add(x)
return uniq,seen
\ No newline at end of file
download:
- implement the work around for AEF50 in the download to make the NUC
- change multiprocessing to multi-threading --> done in Version V2.9.0
- implement multi-emissivity calculation
- implement absolute calibration
- implement download of the stored temperature data (After the upload)
- implement download of the stored heat flux data (After the upload)
- implement download of FLIR data --> Done in V2.8.0, in testing phase
upload:
- upload the new nucing files for AEF50 (the workaround)
- upload remaining data from failed uploads
- upload scene models
- upload temperature
- upload heat flux data
- upload FLIR data
- upload FLIR LUT
This diff is collapsed.
from . import IR_image_tools
from . import downloadversionIRdata
from . import IR_config_constants
\ No newline at end of file
This diff is collapsed.
......@@ -10,12 +10,12 @@ loading IR data and printing plots
import numpy as np
import downloadversionIRdata as downIR
import matplotlib.pyplot as plt
import datetime
if __name__=='__main__':
#%% loading data
status,time,images,valid=downIR.get_temp_from_raw_by_program(20,"20171207.006",time_s=3.5,emi=0.8,T_version=2,version=1,threads=8,give_ERROR=False)
print(datetime.datetime.now(),"start")
status,time,images,valid=downIR.get_temp_from_raw_by_program(10,"20180801.017",time_s=2.5,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,verbose=5)
print('fertig')
#%% plotting data
......@@ -33,5 +33,5 @@ if __name__=='__main__':
labelbottom='off') # labels along the bottom edge are off
c_ax = plt.colorbar()
c_ax.set_label('T [K]')
plt.savefig('20171207.006 - 2.2s - AEF20IR - in K.png', dpi=300, bbox_inches='tight')
plt.savefig('20180726.028 - 2.2s - AEF50IR - in K.png', dpi=300, bbox_inches='tight')
plt.show()
# -*- coding: utf-8 -*-
"""
Created on Thu Nov 29 17:41:40 2018
V3.0.2
@author: holn
"""
import numpy as np
import downloadversionIRdata as IR
import matplotlib.pyplot as plt
import datetime
if __name__=='__main__':
#%% loading data
print(datetime.datetime.now(),"start")
status,times,images=IR.download_heatflux_by_program(20,"20171109.008",timewindow=0.9,version=1,threads=1,verbose=5)
print('done')
#%% plotting data
status2,time,s,q=IR.extract_heatflux_profile_from_DL((np.asarray(times)-times[0])/1e9,images,profile="TM3h_5_5",timewindow=[0,0.5],verbose=5)
plt.figure()
plt.imshow(images[-1]/1e6)
plt.clim([0,4])
plt.tick_params(axis='both', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
left='off',
right='off',
labelleft='off',
labelbottom='off') # labels along the bottom edge are off
c_ax = plt.colorbar()
c_ax.set_label('q [MW/m2]')
plt.savefig('20171109.008 - '+str(round((np.asarray(times[-1])-times[0])/1e9,2))+' - AEF20IR - in MWm2.png', dpi=300, bbox_inches='tight')
if len(np.shape(q))==1:
plt.figure()
plt.plot(s,q/1e6,'ko-')
plt.xlabel("s [m]",fontsize=26)
plt.ylabel("q [MW/m2]",fontsize=26)
plt.tick_params(labelsize=26)
plt.title("20171109.008 heat flux profile for TM3, finger 5, profile 5 @1s",fontsize=26)
plt.grid()
plt.ylim(bottom=0)
plt.show()
elif len(np.shape(q))==2:
plt.figure()
X,Y=np.meshgrid(time,s)
plt.pcolor(X,Y,q.swapaxes(0,1)/1e6,vmin=0)
plt.xlabel("time [s]",fontsize=26)
plt.ylabel("s [m]",fontsize=26)
plt.tick_params(labelsize=26)
cb=plt.colorbar()
cb.set_label('heatflux in MW/m²', rotation=270, labelpad=15,fontsize=26)
cb.ax.tick_params(labelsize=26)
plt.title("20171109.008 heat flux profile for TM3, finger 5, profile 5",fontsize=26)
plt.show()
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment