Skip to content

GitLab

Commit 9d56fd37 authored by Holger Niemann's avatar Holger Niemann
Browse files

Merge branch 'Holgers' into 'master' 

Holgers

See merge request holn/IR_data_access!46
parents 44885d00 03acd06d
Hide whitespace changes
Inline Side-by-side
IR_image_tools.py
View file @ 9d56fd37
# -*- coding: utf-8 -*-
"""
Created on Wed May 9 14:56:32 2018
Version: 3.4.0
Version: 3.4.2
@author: Holger Niemann, Peter Drewelow, Yu Gao
mainly to clean up the downloadversionIRdata code
......@@ -35,9 +35,14 @@ def get_OP_by_time(time_ns=None, shot_no=None, program_str=None):
e.g. '20171207.022' or '20171207' (OPTIONAL)
RESULT
------
conn - MDSplus connection object, to be used in e.g. 1511972727249834301
read_MDSplus_image_simple(), read_MDSplus_metadata()
OP: string
the operation phase as a string
NOTE
------
Needs to be updated for OP2.0 and OP2.X as well for OP3, 4 etc.
'''
#conn - MDSplus connection object, to be used in e.g. 1511972727249834301
# read_MDSplus_image_simple(), read_MDSplus_metadata()
# derive operation phase (OP) from time as nanosecond time stamp or string
if time_ns is not None:
dateOP = datetime.datetime.utcfromtimestamp(time_ns/1e9)
......@@ -70,10 +75,14 @@ def bestimmtheitsmass_general(data, fit):
"""
INPUT
------
data: list or array
your 1D data
fit: list or array
your 1D fit data
RESULT
------
R2: float
the regression value
NOTE
------
"""
......@@ -90,10 +99,14 @@ def bestimmheitsmass_linear(data, fit, debugmode=False):
"""
INPUT
------
data: list or array
your 1D data
fit: list or array
your 1D fit data
RESULT
------
R2: float
the regression value
NOTE
------
"""
......@@ -113,10 +126,14 @@ def quad_abweich_mittel(data, mittel):
"""
INPUT
------
data: list or array
the data as 1D list or array
mittel: value
the mean value of the data
RESULT
------
R: float
the quadratic mean difference
NOTE
------
"""
......@@ -129,10 +146,14 @@ def quad_abweich(data, fit):
"""
INPUT
------
data: list or array
the data as 1D list or array
fit: list or array
the fit of the data, same length as data
RESULT
------
R: float
the quadratic difference
NOTE
------
"""
......@@ -146,14 +167,21 @@ def quad_abweich(data, fit):
def find_nearest(array, value):
"""
find the nearest/closest value in the array and returns the index of it.
INPUT
------
array: list or array
the list with all values, 1D
value: integer or float
the value
RESULT
------
idx: integer
index of the value closest to the requested one
NOTE
------
it could work also for strings, but was not tested for string values
"""
#a=array
a = [x - value for x in array]
......@@ -168,10 +196,20 @@ def check_coldframe(coldframe, references=None, threshold=0.5, plot_it=False):
INPUT
------
coldframe: 2D numpy array
the cold frame, which should be tested
references: float, default None
the reference value, the data in the image should be compared to
threshold: float, optional, default 0.5
sets the threshold for the minimum quality factor which should be reached
plot_it: boolean, default False,
if True the result of the bad pixel finding will be plotted
RESULT
------
valid: boolean
True if the cold frame can be used
R: float
the quality factor
NOTE
------
'''
......@@ -211,10 +249,20 @@ def check_coldframe_by_refframe(coldframe, reference_frame, threshold=0.8, plot_
'''
INPUT
------
coldframe: 2D numpy array
the cold frame, which should be tested
reference_frame: 2D numpy array
the reference frame, the data in the image should be compared to
threshold: float, optional, default 0.8
sets the threshold for the minimum quality factor which should be reached
plot_it: boolean, default False,
if True the result of the bad pixel finding will be plotted
RESULT
------
valid: boolean
True if the cold frame can be used
R: float
the quality factor
NOTE
------
'''
......@@ -231,10 +279,16 @@ def check_backgroundframe(backgroundframe, threshold=50):
INPUT
------
backgroundframe: 2d numpy array
the background frame, which should be tested
threshold: float or integer, default 50
the maximum threshold for the mean of the image
RESULT
------
valid: boolean
True if the image is good
mean: float
the mean value of the data
NOTE
------
'''
......@@ -251,22 +305,22 @@ def check_backgroundframe(backgroundframe, threshold=50):
def read_bad_pixels_from_file(port, shot_no=None, program=None, time_ns=None):
'''Reads bad pixels stored in *.bpx file on E4 server.
Requires one of the optional arguments shot_no or program.
IN
port - integer of port no of camera
shot_no - integer of MDSplus style shot number, e.g. 171207022 (OPTIONAL)
program - string of CoDaQ ArchiveDB style prgram number or date,
e.g. '20171207.022' or '20171207' (OPTIONAL)
OUT
bad_pixle_list - list of tuples (row,column) of pixel coordinates
as integer
Requires one of the optional arguments shot_no or program.
INPUT
------
port: integer
integer of port no of camera
shot_no: integer
integer of MDSplus style shot number, e.g. 171207022 (OPTIONAL)
program: string
string of CoDaQ ArchiveDB style prgram number or date,\n
e.g. '20171207.022' or '20171207' (OPTIONAL)
RESULT
------
bad_pixle_list: list of tuples (row,column) of pixel coordinates as integer
the bad pixel list
NOTE
------
'''
......@@ -295,10 +349,17 @@ def find_outlier_pixels(frame, tolerance=3, plot_it=False):#worry_about_edges=Tr
INPUT
------
frame: numpy array, 2D
the image, where to find the bad pixels in
tolerance: integer, optional, default 3
the tolerance level for the detection of the outlier.\n
defines how many sigmas the outlier has be out of the mean value of the image
plot_it: boolean
for True, the histogramm of the analysis will be plotted, for False not plotting
RESULT
------
bad_pixels: list
the list of bad pixels as tuples (row,column) of pixel coordinates as integer
NOTE
------
'''
......@@ -339,12 +400,21 @@ def find_outlier_pixels(frame, tolerance=3, plot_it=False):#worry_about_edges=Tr
def correct_images(images, badpixels, verbose=0):
'''
applies the bad pixel correction onto the given images, based on the given bad pixel list
INPUT
------
images: list of 2D numpy array or 3D numpy array, (first dimension time)
the images where a bad pixel correction should be applied on, first dimension is time
badpixels: list or 2D numpy array
either list of tuples (row,column) of bad pixel coordinates as integer values,\n
or mask of pixel status (good=True, bad=False)
verbose: integer, optional, default 0
feedback level (details of print messages)
RESULT
------
images: list of 2D numpy array or 3D numpy array(first dimension time)
the images with corrected bad pixels
NOTE
------
'''
......@@ -540,12 +610,21 @@ def restore_bad_pixels(frames, bad_pixel, by_list=True, check_neighbours=True, p
def generate_new_hot_image(cold,reference_cold,reference_hot):
'''
generates on the basis of the reference hot and cold frame and a given cold frame a new hot frame for a two point NUC
INPUT
------
cold: numpy array, 2D
the cold frame from the camera (an image of a uniform, flat cold object, e.g. a closed shutter)
reference_cold: numpy array, 2D
the reference cold frame from the calibration or from the company
reference_hot: numpy array, 2D
the reference hot frame from the calibration or from the company
RESULT
------
hot_image: numpy array, 2D
generated hot image
NOTE
------
'''
......@@ -556,12 +635,30 @@ def generate_new_hot_image(cold,reference_cold,reference_hot):
def calculate_gain_offset_image_pix(cold_image,hot_image=None,reference_cold=None,reference_hot=None,verbose=0):
'''
will calculate the gain and offset image for a given cold and hot image with respect to the center pixel. \n
if the hot image cannot be provided, the code will calculate it based on the given reference cold and refrence hot images
INPUT
------
cold_image: numpy array, 2D
the cold frame from the camera (an image of a uniform, flat cold object, e.g. a closed shutter)
hot_image: numpy array, 2D, optional, default None
the hot frame from the cmaera (an image of a uniform, flat hot object, e.g. flat hot source or a closed, heated shutter)\n
will be generated if not give, see generate_new_hot_image
reference_cold: numpy array, 2D, optional, default None
the reference cold frame from the calibration or from the company\n
only needed if no hot image is given, see generate_new_hot_image
reference_hot: numpy array, 2D, optional, default None
the reference hot frame from the calibration or from the company\n
only needed if no hot image is given, see generate_new_hot_image
verbose: integer, optional, default 0
feedback level (details of print messages)
RESULT
------
Gain: numpy array, 2D
relative gain image for the non-uniformity correction
Offset: numpy array, 2D
relative offset image for the non-uniformity correction
NOTE
------
'''
......@@ -579,12 +676,30 @@ def calculate_gain_offset_image_pix(cold_image,hot_image=None,reference_cold=Non
def calculate_gain_offset_image(cold_image,hot_image=None,reference_cold=None,reference_hot=None,verbose=0):
"""
will calculate the gain and offset image for a given cold and hot image with respect to the mean of the center pixels(5x5). \n
if the hot image cannot be provided, the code will calculate it based on the given reference cold and refrence hot images
INPUT
------
cold_image: numpy array, 2D
the cold frame from the camera (an image of a uniform, flat cold object, e.g. a closed shutter)
hot_image: numpy array, 2D, optional, default None
the hot frame from the cmaera (an image of a uniform, flat hot object, e.g. flat hot source or a closed, heated shutter)\n
will be generated if not give, see generate_new_hot_image
reference_cold: numpy array, 2D, optional, default None
the reference cold frame from the calibration or from the company\n
only needed if no hot image is given, see generate_new_hot_image
reference_hot: numpy array, 2D, optional, default None
the reference hot frame from the calibration or from the company\n
only needed if no hot image is given, see generate_new_hot_image
verbose: integer, optional, default 0
feedback level (details of print messages)
RESULT
------
Gain: numpy array, 2D
relative gain image for the non-uniformity correction
Offset: numpy array, 2D
relative offset image for the non-uniformity correction
NOTE
------
"""
......@@ -639,20 +754,27 @@ def calculate_gain_offset_image(cold_image,hot_image=None,reference_cold=None,re
def reconstruct_coldframe (exposuretime, sT, a, bnew, coldref):
"""
will generate a coldframe, based of the reference cold frame and a fit, based on the sensor temperature and exposure time.
INPUT
------
exposuretime: integer
the exposure time
sT:
a:
bnew:
sT: float
sensor temperature, see get_sensor_temp_by_program
a: numpy array
fit coefficient image A for the reconstruction, see the files on the E4-Server
bnew: numpy array
fit coefficent image B for the reconstruction, see the files on the E4-Server
coldref: numpy array
the reference cold frame as the base for the reconstruction
the reference cold frame as the base for the reconstruction, see download_hot_cold_reference_by_times
RESULT
------
cirebiuld: numpy array
the reconstructed cold frame
NOTE
------
the fitting coefficients have been calculated from several known cold frames for different sensors temperature, the procedure have been applied for each camera separatly
"""
cirebuild = a * sT + bnew * exposuretime + coldref
return cirebuild
......@@ -661,12 +783,16 @@ def reconstruct_coldframe (exposuretime, sT, a, bnew, coldref):
#%% other functions
def check_dublicates(array):
"""
checks the given array for dublicates and gives pack the dublicates
INPUT
------
array: list or array, 1D
the list to check for dublicates
RESULT
------
dublicates: list
list of the values which appear more than ones in the array
NOTE
------
"""
......@@ -676,12 +802,18 @@ def check_dublicates(array):
def check_dublicates_2(array):
"""
checks the given array for dublicates and gives pack the list with removed dublicates in original order and sorted
INPUT
------
array: list or array, 1D
the list to check for dublicates
RESULT
------
uniq: list
the array in orignal order with removed dublicates
seen: list, set
the array in sorted order with removed dublicates
NOTE
------
"""
......@@ -725,13 +857,13 @@ def get_work_list(pipepath,typ="q"):
koline=line.split("\t")
if len(koline)>1:
prog = koline[0]
if typ in ('q','load'):
if typ in ('q','load','qpeak','Aw','width','q_old','load_old'):
cam_programs.append((prog,koline[1].split("\n")[0]))
else:
cam_programs.append((prog,koline[1]))
reasons.append(koline[2].split("\n")[0])
f.close()
if typ in ('q','load'):
if typ in ('q','load','qpeak','Aw','width','q_old','load_old'):
bla=check_dublicates_2(cam_programs)
cam_programs=bla[0]
return cam_programs
......@@ -1771,14 +1903,14 @@ def convert_Bad_Pixels(Badpixels,map2D):
INPUT
-----
Badpixels: 2D array
image of the bad pixels with 0 no bad pixel and >0 as bad pixel
map2D: dictonary of the
see downloadversionIRdata.download_heatflux_scene_model_reference()
Badpixels: 2D array
image of the bad pixels with 0 no bad pixel and >0 as bad pixel
map2D: dictonary of the
see downloadversionIRdata.download_heatflux_scene_model_reference()
RESULT
-----
heatflux_badpixel: numpy array
good pixels have a zero, bad pixels have a 1
heatflux_badpixel: numpy array
good pixels have a zero, bad pixels have a 1
"""
heatflux_badpixel=np.zeros(np.shape(map2D['Pixel_Y']))
PX=map2D['Pixel_X']
......
README.md
View file @ 9d56fd37
This collections of functions and tools are used to download and evaluated data related to the Infrared cameras, including thermocouples.
To install this libary, run the setup
python install setup.py
\ No newline at end of file
python setup.py install
\ No newline at end of file
downloadversionIRdata.py
View file @ 9d56fd37
......@@ -4,7 +4,7 @@ Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018
last update on Fr Nov 23 15:37:00 2018
Version: 3.4.1
Version: 3.4.2
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3)
.
#of updates to add functionalities
......@@ -12,7 +12,7 @@ Version: 3.4.1
#number of updates for bug fixes )
@author: holn
"""
version = "V3.4.1"
version = "V3.4.2"
import numpy as np
import IR_image_tools as IR_tools
......@@ -55,6 +55,18 @@ except:
config_path = ""
def get_started():
"""
call it to get a short introduction to the available functions in this library\n
it opens a weblink to a presentation and as well a flow chart
"""
print("Welcome")
import webbrowser
webbrowser.open('https://wikis.ipp-hgw.mpg.de/PhysicsW7X/images/9/9f/2019-04-24_20190424_How_to_access_Infrared_Data.pdf', new=2)
path=str(repr(__file__)).split("downloadversionIRdata")[0].split("'")[1]
os.startfile(path+"IR_data_access_flow_chart.png")
def get_latest_version(stream, project="W7X", testmode=False, t_from=None,
t_to=None, program=None, verbose=0):
"""Find out latest version of given stream
......@@ -249,7 +261,9 @@ def download_LUT(port, time, exposure=0, emissivity=0, camera_filter=0, version=
LUTpar = read_restdb_old(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
if LUTpar[0]:
LUTid = LUTpar[2][0]['structure'][query]
LUTs = read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(time-10)+"&upto="+str(time+20))
#getting here now the time from the parlog
Timetamp = LUTpar[2][0]['structure']['timestamp']
LUTs = read_restdb_old(larchivepath+"DATASTREAM/V"+str(version)+"/0/LUT/_signal.json?from="+str(Timetamp-10)+"&upto="+str(Timetamp+20))
if LUTs[0]:
LUTs = LUTs[2][0]#.swapaxes(1,2)[0] fixed, somehow the archive gives now data in a swaped way back
LUT = [LUTs[0], LUTs[LUTid], LUTs[LUTid+1]]
......@@ -5396,7 +5410,7 @@ def extract_temperature_profile_from_DL(port, time, images, profile, finger=None
time_window=None, inputchecked=False,
verbose=0, reference_time=0):
"""
for a given heatflux image and profile and finger information, a profile can be extracted.
for a given temperature image and profile and finger information, a profile can be extracted.
INPUT
------
port: int or str
......@@ -5404,7 +5418,7 @@ def extract_temperature_profile_from_DL(port, time, images, profile, finger=None
time: array or numpy array
the time vector for the given images
images: numpy array
temperature flux images, from which the profile should be extracted
temperature images, from which the profile should be extracted
profile: string, integer or float,
string: "TM3h_5_5" or "1605" or "16.05" or "5", similar for float and integer. single number only if the finger is given!
finger: string or integer,
......@@ -6122,7 +6136,7 @@ def get_trigger_from_PID(program, port=None, testmode=False, verbose=0):
if __name__=='__main__':
print("local function calling")
get_started()
#%% temperature download and plotting example
# port=31#"AEF50"#"AEF51"'20171114.053'#
# prog="20180911.008"#"20181011.033"#'20171122.035'#'20181011.010'#'20180823.037'#'20170927.020'#"20181011.036"#"20181016.037"#"20180920.042"#"20171109.021"#"20181010.036"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment