Commit fb96987d authored by Peter Drewelow's avatar Peter Drewelow
Browse files

Merge branch 'master' of gitlab.mpcdf.mpg.de:holn/IR_data_access into peter

parents dbe0e27f 5a0379bb
<<<<<<< HEAD
26.03.2019: Upadte to V3.2.0:
- scne model download possible
- scene model reference download possible for heat flux data
- downsampled temperature image stream possible
- extration of temperature profiles possible
27/28/29.11.2018: update to V3
- add file upload_heatflux.py
- add plot_heatflux_example.py
......@@ -44,6 +48,7 @@ bugfixes in downloadversionIRdata:
- fixed: wrong time intervall for TC divertor data
Versions:
V3.2.0: download of scene models possible, temperature profiles can be extracted, downsampled temperature images available
V3.1.0: add of divertor loads in the upload and the download
V3.0.2: change the upload of heat flux from float64 to float32. extract profile accepts now also timewindows.
V3.0.1: extension of the heat flux upload (scene model reference can be uploaded)
......
......@@ -70,6 +70,18 @@ def bestimmtheitsmass_general(data,fit):
else:
print("bestimmtheitsmass_general: Arrays must have same dimensions")
return R
def bestimmheitsmass_linear(data,fit,debugmode=False):
R2=0
if len(fit)==len(data):
mittel_D=np.mean(data)#np.sum(data)/len(data)
mittel_F=np.mean(fit)
R2=quad_abweich_mittel(fit,mittel_D)/quad_abweich_mittel(data,mittel_D)
if debugmode:
print(mittel_D,mittel_F,quad_abweich_mittel(fit,mittel_D),quad_abweich_mittel(data,mittel_D),R2)
else:
print("bestimmtheitsmass_linear: Arrays must have same dimensions")
return R2
def quad_abweich_mittel(data,mittel):
R=0
......
......@@ -6,14 +6,3 @@ download:
- implement download of the stored temperature data (After the upload)
- implement download of the stored heat flux data --> done in V3.0.0
- implement download of FLIR data --> Done in V2.8.0, in testing phase
upload:
- upload the new nucing files for AEF50 (the workaround)
- upload remaining data from failed uploads
- upload scene models
- upload temperature
- upload heat flux data (started)
- upload FLIR data (partly done)
- upload FLIR LUT (partly done)
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 26 16:01:06 2017
Version: 2.9.4
@author: holn
"""
import uploadingversionIRdata as IRup
import os
import h5py as h5
import numpy as np
import datetime
import sys
from time import sleep
from os.path import join
#from getfoldernames import Searchdatalocation as gfn
#%% set global parameter
### default old settings, will be overrided by the loaded config file!!! ###
local= True # local test on one IRcam PC (not central PC)
active_PC = [0] # selection of the following camera PCs to loop over
delayupload=True
startuploadtime="20:00:00"
reupload=False
reason=""
retry=2
port="AEF10"
singleupload=False#True#False
singlelist=[]
raw_parlog=True
raw_data=True#False#True
raw1=True#False#True#False#True
raw2=True
back_parlog=True#False#True
background=True#False#True
nuc_parlog=True#False
nuc=True
metastream=True
dates=[[2017,11,9]]
temperature=False
only_upload_lattest=False
use_today=False
delete_Files=False
logpath=""#"C:\\QIR\\Upload\\"
temppath=""#"C:\\QIR\\Upload\\"
ports = ['AEF10', 'AEF11',
'AEF20', 'AEF21',
'AEF30', 'AEF31',
'AEF40', 'AEF41',
'AEF50', 'AEF51','AEK51']
cameras = ['IRCAM', 'IRCAM',
'IRCAM', 'IRCAM',
'IRCAM', 'IRCAM',
'IRCAM', 'IRCAM',
'INFRATEC', 'IRCAM','FLIR']
Tup = [ False, True,
True, True,
True, False,
False, True,
False, True]
pc_list = ['Pc-e4-qrt-7', 'Pc-e4-qrt-2',
'Pc-e4-qrt-3', 'Pc-e4-qrt-4',
'Pc-dia-ws-16', 'Pc-e4-qrt-6',
'Pc-dia-ws-17', 'Pc-dia-ws-18',
'Pc-e4-qrt-5', 'Pc-e4-qrt-1','PC-HC-MESS-2']
try:
cFile=open("upload_config")
for line in cFile:
if line[0:3]!="###":
vars()[line.split("=")[0]]=eval(line.split("=")[1].split("\n")[0])
cFile.close()
if use_today:
dummy=datetime.datetime.now()
dates=[[dummy.year,dummy.month,dummy.day]]
except Exception as E:
raise Exception("Config file was not loaded properly",E)
def findthelatestfilename(Rawdatapath):
'''
find the latest modified file name for easy upload.
@Rawdatapath: the path to search from, e.g: 'R:\\2018.06.07'.
return the latest modified file name in string.
'''
filelist = os.listdir(Rawdatapath)
timearray = [os.path.getmtime(join(Rawdatapath, i)) for i in filelist]
maxtimestampindex = np.argmax(timearray)
latestfilename = filelist[maxtimestampindex]
return latestfilename
#create upload log file:
class Tee(object):
def __init__(self, *files):
self.files = files
def write(self, obj):
for f in self.files:
f.write(obj)
f.flush() # If you want the output to be visible immediately
def flush(self) :
for f in self.files:
f.flush()
if delayupload:
now=datetime.datetime.now()
start=datetime.datetime(now.year,now.month,now.day,int(startuploadtime.split(":")[0]),int(startuploadtime.split(":")[1]),int(startuploadtime.split(":")[2]))
sleeptime=(start-now).total_seconds() #seconds
if sleeptime<0:
sleeptime=1
sleep(sleeptime)
original = sys.stdout
nrprograms=0
fully_uploaded=0
sum_meta1=0
sum_meta2=0
sum_raw1=0
sum_raw2=0
start=datetime.datetime.now()
fout=open(logpath+"upload_log_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
#foutf=open("upload_log_failes_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
sys.stdout = Tee(sys.stdout, fout)
#sys.stdout = fout
f_success=open(logpath+"upload_log_successlist_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
try:
try:
sysport=os.environ['qir_port']
except Exception as E:
sysport=port
print("unable to read the port from the system, using config port")
raise Exception(E)
toff = 0 #delay time of the programm for the raw stuff in ms
port=sysport
active_PC=[ports.index(port)]
Tup[ports.index(port)]=temperature
#%% decide data path format
for date in dates:
if date[0]==2017 and date[1]<=9 and date[2]<=25:
# old file and folder names
rawdatapath = "\\\\{0:s}\\r\\{1[0]:4d}-{1[1]:02d}-{1[2]:02d}"
calibdatapath = "\\\\{0:s}\\r\\Calibration\\{1[0]:4d}-{1[1]:02d}-{1[2]:02d}"
tempdatapath = "\\\\{0:s}\\t\\{1[0]:4d}-{1[1]:02d}-{1[2]:02d}"
"""
have to be finished!!
"""
else:
# recent file and folder naming convention
rawdatapath = "\\\\{0:s}\\r\\{1[0]:4d}.{1[1]:02d}.{1[2]:02d}"
calibdatapath = rawdatapath
tempdatapath = "\\\\{0:s}\\t\\{1[0]:4d}.{1[1]:02d}.{1[2]:02d}"
if local:
rawdatapath = "R:\\{1[0]:4d}.{1[1]:02d}.{1[2]:02d}"
calibdatapath = rawdatapath
tempdatapath = "T:\\{1[0]:4d}.{1[1]:02d}.{1[2]:02d}"
#%% loop over different IRcamera PC and upload data
if not raw1 and not raw2:
raw_parlog=False
raw_data=False
for i in active_PC:
if only_upload_lattest:
single_upload=True
singlelist = [findthelatestfilename(rawdatapath.format(pc_list[i],date))]
port = int(ports[i].split('AEF')[1])
path = rawdatapath.format(pc_list[i],date)
print(datetime.datetime.now(),"Starting Upload for "+ports[i],date)
f_success.write(ports[i]+" \n")
f_success.flush()
f_success.write("program folder meta1 meta2 raw1 raw2 \n")
f_success.flush()
# foutf.write(datetime.datetime.now(),"Starting Upload for "+ports[i],date,"\n")
# find all programs of this date on this PC
programtimes = []
if singleupload:
programtimes=singlelist
else:
for o in os.listdir(path):
if os.path.isdir(os.path.join(path,o)):
if len(o.split("_"))>1 and not date[0]>=2018:
print("unsupported folderstructure at the moment!!")
else:
programtimes.append(o)
# # loop over all programs and upload data
for prog_time in programtimes:
nrprograms+=1
# loop over one specific program for test and upload data
print("start upload of files for "+str(prog_time)+" for date: ",date)
# foutf.write("start upload of files for "+str(prog_time)+" for date: ",date,"\n")
try:
path = os.path.join(rawdatapath.format(pc_list[i],date), prog_time)
# sort files into background, cold image, offset, gain and raw data
filelist = os.listdir(path)
blist = {} # background dic
clist = {} # cold file dic
olist = {} # offset file dic
glist = {} # gain file dic
rlist = {} # raw file dic
if len(filelist)>0:# check for empty folder!
for file in filelist:
dummy = file.split("_")
if date[0]>=2018:
if dummy[4]=='raw' or dummy[4]=='cal':
rlist[dummy[1]]=file
else:
if dummy[1]=='background':
blist[dummy[7]] = file
elif dummy[1]=='nuc':
if dummy[2]=='cold':
clist[dummy[8]] = file
elif dummy[2]=='gain':
glist[dummy[8]] = file
elif dummy[2]=='offset':
olist[dummy[8]] = file
elif dummy[1]=='raw':
rlist[dummy[3]] = file
else:
print("unexpected file in the folder!")
# foutf.write(datetime.datetime.now(),"unexpected file in the folder",prog_time,"\n")
print(datetime.datetime.now()," Start NUC upload")
# upload of NUC files, background files
if cameras[i]=="IRCAM":
newversion=reupload
if background or nuc:
for t_exp in range(1,10):
try:
print(datetime.datetime.now()," prepare upload of nuc data set for "+str(t_exp)+"us")
fb = h5.File(os.path.join(path, blist[str(t_exp)+"us"]),'r')
background_image = np.asarray(fb[list(fb.keys())[0]])
fb.close()
fg = h5.File(os.path.join(path, glist[str(t_exp)+"us"]),'r')
gain_image = np.asarray(fg[list(fg.keys())[0]])
fg.close()
fo = h5.File(os.path.join(path, olist[str(t_exp)+"us"]),'r')
offset_image = np.asarray(fo[list(fo.keys())[0]])
fo.close()
fc = h5.File(os.path.join(path, clist[str(t_exp)+"us"]),'r')
cold_image = np.asarray(fc[list(fc.keys())[0]])
fc.close()
bad = []
ftime = blist[str(t_exp)+"us"].split("_")[3].split(".")
ftime = list(map(int, ftime))
ftime[3] = ftime[3]*1000 #µs not ms!
time_stamp = int(IRup.TimeToNs(date,ftime))
#time_stamp = time_stamp #+ 6 * 6E10
if background:
try:
IRup.upload_Background_frame(cameras[i], port, [time_stamp], [t_exp],
background_image, newversion=newversion,reason=reason)
except Exception as E:
print("upload of background files for "+str(t_exp)+"us failed")
print(E)
# foutf.write(datetime.datetime.now(),"upload of background files for "+str(t_exp)+"us failed",E,"\n")
else:
print("background upload deactivated")
# foutf.write(datetime.datetime.now(),"background upload deactivated","\n")
if nuc:
try:
IRup.upload_NUC_ref(cameras[i], port, [time_stamp], [t_exp],
gain_image, offset_image, cold_image, bad,parlog=nuc_parlog,data=nuc,newversion=newversion,reason=reason)
except Exception as E:
print("upload of NUC files for "+str(t_exp)+"us failed")
print(E)
# foutf.write(datetime.datetime.now(),"upload of NUC files for "+str(t_exp)+"us failed",E,"\n")
else:
print("nuc upload deactivated")
# foutf.write(datetime.datetime.now(),"nuc upload deactivated","\n")
newversion=False#otherwise 6 new versions!
except Exception as E:
print(E)
# foutf.write(datetime.datetime.now(),"error in reading",E,"\n")
else:
print("background and NUC upload deactivated")
# foutf.write(datetime.datetime.now(),"background and NUC upload deactivated","\n")
# -->| tested --> works for IRCAM
else: #case for the Infratec files
newversion=reupload
if background:
fkeys=list(blist.keys())
print(datetime.datetime.now()," prepare upload of background data set for "+str(fkeys[0]))
filename=blist[fkeys[0]]
fb = h5.File(os.path.join(path, filename),'r')
background_image = np.asarray(fb[list(fb.keys())[0]])
ftime = filename.split("_")[3].split(".")
ftime = list(map(int, ftime))
ftime[3] = ftime[3]*1000 #µs not ms!
time_stamp = IRup.TimeToNs(date,ftime)
t_exp=int(filename.split("_")[7].split("us")[0])
cfilter=int(filename.split("_")[5])
try:
IRup.upload_Background_frame(cameras[i], port, [time_stamp], [t_exp,cfilter],
background_image, newversion=newversion,reason=reason)
except Exception as E:
print("upload of background file for "+str(t_exp)+"us failed")
print(E)
# foutf.write(datetime.datetime.now(),"upload of background file for "+str(t_exp)+"us failed",E,"\n")
else:
print("background upload deactivated")
# foutf.write(datetime.datetime.now(),"background upload deactivated","\n")
newversion=reupload
#%% ###upload now raw files###
fkeys=list(rlist.keys())
ftime1=fkeys[0].split(".")
ftime1 = list(map(int, ftime1))
ftime1[3]=ftime1[3]*1000#µs not ms!
ftime2=fkeys[1].split(".")
ftime2 = list(map(int, ftime2))
ftime2[3]=ftime2[3]*1000#µs not ms!
t1=int(IRup.TimeToNs(date,ftime1))
t2=int(IRup.TimeToNs(date,ftime2))
if t1>t2:
f1=1
f2=0
dummy=t2
t2=t1
t1=dummy
else:
f1=0
f2=1
# bla='09.34.44.100000'#'09.29.23.172000'
# bla=bla.split(".")
# bla = list(map(int, bla))
# t2=int(IRup.TimeToNs(date,bla))
hf1=h5.File(path+"\\"+rlist[fkeys[f1]],'r')
attri1=hf1['dynamic_attributes']
metadata1={
'serialnumer': rlist[fkeys[f1]].split("_")[-1].split(".")[0],
'width':int(np.asarray(attri1['ROI_w'])[0]),
'heigth':int(np.asarray(attri1['ROI_h'])[0]),
'pos_left':int(np.asarray(attri1['ROI_x'])[0]),
'pos_top':int(np.asarray(attri1['ROI_y'])[0]),
'bitdepth': int(np.asarray(attri1['bitdepth'])[0])
}
# tmeta1={'divertortemperature_K':np.asarray(attri1['divertor_temp_K'])[0],
# 'timestamp':np.asarray(attri1['timestamp'])[0]}
time1=np.asarray(hf1['timestamps'])
sensor1=np.asarray(attri1['sensor_temp_C'])
chip1=np.asarray(attri1['camera_temp_C'])
framerate1=np.asarray(attri1['frame_rate'])
expo1=np.asarray(attri1['integration_time_us'])
hf2=h5.File(path+"\\"+rlist[fkeys[f2]],'r')
attri2=hf2['dynamic_attributes']
metadata2={
'serialnumer': rlist[fkeys[f1]].split("_")[-1].split(".")[0],
'width':int(np.asarray(attri2['ROI_w'])[0]),
'heigth':int(np.asarray(attri2['ROI_h'])[0]),
'pos_left':int(np.asarray(attri2['ROI_x'])[0]),
'pos_top':int(np.asarray(attri2['ROI_y'])[0]),
'bitdepth': int(np.asarray(attri2['bitdepth'])[0])
}
# tmeta2={'divertortemperature_K':np.asarray(attri2['divertor_temp_K'])[0],
# 'timestamp':np.asarray(attri2['timestamp'])[0]}
time2=np.asarray(hf2['timestamps'])
sensor2=np.asarray(attri2['sensor_temp_C'])
chip2=np.asarray(attri2['camera_temp_C'])
framerate2=np.asarray(attri2['frame_rate'])
expo2=np.asarray(attri2['integration_time_us'])
if cameras[i]=="IRCAM":
metadata1['cameraname']= 'IRCam_Caleo768kL'
metadata2['cameraname']= 'IRCam_Caleo768kL'
metadata1['purpose']='NUC+background'
metadata2['purpose']='experiment'
#['ROI_h', 'ROI_w', 'ROI_x', 'ROI_y', 'Size',
#'bitdepth', 'camera_temp_C', 'divertor_temp_K', 'frame_number',
#'frame_rate', 'integration_time_us', 'sensor_temp_C', 'timestamp']
elif cameras[i]=="INFRATEC":
metadata1['cameraname']= 'INFRATEC_IMAGEIR9300'
metadata2['cameraname']= 'INFRATEC_IMAGEIR9300'
metadata1['purpose']='background'
metadata2['purpose']='experiment'
#metadata2['multi_exposure']=np.asarray(attri2['multi_exposure'])[0]
#metadata1['multi_exposure']=np.asarray(attri1['multi_exposure'])[0]
elif cameras[i]=="FLIR":
metadata1['cameraname']= 'FLIR_SC8303_00037'
metadata2['cameraname']= 'FLIR_SC8303_00037'
metadata1['purpose']='background'
metadata2['purpose']='experiment'
else:
raise Exception("unknown camera")
if date[0]>=2018:
metadata1['software_version']=hf1['images'].attrs['software_version'].decode("UTF-8")
metadata1['filter']=int(np.asarray(attri1['filter'])[0])
metadata1['view_flip_h']=bool(np.asarray(attri1['view_flip_h'])[0])
metadata1['view_flip_v']=bool(np.asarray(attri1['view_flip_v'])[0])
metadata2['software_version']=hf2['images'].attrs['software_version'].decode("UTF-8")
metadata2['filter']=int(np.asarray(attri2['filter'])[0])
metadata2['view_flip_h']=bool(np.asarray(attri2['view_flip_h'])[0])
metadata2['view_flip_v']=bool(np.asarray(attri2['view_flip_v'])[0])
metadata1['cameraname']= hf1['images'].attrs['camera'].decode("UTF-8")
metadata2['cameraname']= hf2['images'].attrs['camera'].decode("UTF-8")
framenumber1=np.asarray(attri1['frame_number'])
framenumber2=np.asarray(attri2['frame_number'])
try:
frametype1=np.asarray(attri1['frame_type'])
# frametype1=[]
# for ele in ftype1:
# if ele == "nuc":
# frametype1.append(0)
# elif ele == "background":
# frametype1.append(1)
# elif ele == "discharge":
# frametype1.append(2)
# else:
# frametype1.append(-1)
except:
frametype1=np.zeros((len(framenumber1)))-1
try:
frametype2=np.asarray(attri2['frame_type'])
# frametype2=[]
# for ele in ftype2:
# if ele == "nuc":
# frametype2.append(0)
# elif ele == "background":
# frametype2.append(1)
# elif ele == "discharge":
# frametype2.append(2)
# else:
# frametype2.append(-1)
except:
frametype2=np.zeros((len(framenumber2)))-1
wrong1,correction_list1=IRup.check_timevector_framenumbers(time1,framenumber1)
wrong2,correction_list2=IRup.check_timevector_framenumbers(time2,framenumber2)
time1_0=time1.copy()
time2_0=time2.copy()
if cameras[i]=="FLIR":
wrong3,correction_list3=IRup.check_extra_frames_FLIR(time2,framenumber2)
wrong2=wrong2 or wrong3
correction_list2=correction_list3+correction_list2
if wrong1:
sensor1=np.delete(sensor1,correction_list1)
chip1=np.delete(chip1,correction_list1)
framerate1=np.delete(framerate1,correction_list1)
frametype1=np.delete(frametype1,correction_list1)
expo1=np.delete(expo1,correction_list1)
time1=np.delete(time1,correction_list1)
if wrong2:
sensor2=np.delete(sensor2,correction_list2)
chip2=np.delete(chip2,correction_list2)
framerate2=np.delete(framerate2,correction_list2)
frametype2=np.delete(frametype2,correction_list2)
expo2=np.delete(expo2,correction_list2)
time2=np.delete(time2,correction_list1)
channel_names=["sensortemperature","cameratemperature","framerate","exposuretime","framenumber","frametype"]
channel_units=["oC","oC",'Hz','us','none','none']
channel_values1=[sensor1, chip1,framerate1,expo1,framenumber1,frametype1]
channel_values2=[sensor2, chip2,framerate2,expo2,framenumber2,frametype2]
comment_meta1="frametype: 0 cold, 1 background, 2 discharge, -1 unknown"
comment_meta2="frametype: 0 cold, 1 background, 2 discharge, -1 unknown"
triggered=np.asarray(attri2['triggered'])
trig_index=np.where(triggered==1)
if len(trig_index[0])>0:
trig_index=min(trig_index[0])
toff=(time2[0]-time2[trig_index])/1e6
else:
print("triggered frame was not found! Assumeing that frame 0 is the t1 frame")
toff=0
print(datetime.datetime.now()," regenerate timevectors")
time1=IRup.regenerate_timevector(time1,0,t2,0)
time2,Program=IRup.regenerate_timevector(time2,1,t2,toff,True)
time1_0=IRup.regenerate_timevector(time1_0,0,t2,0)
time2_0=IRup.regenerate_timevector(time2_0,1,t2,toff)
else:
correction_list1=[]
correction_list2=[]
channel_names=["sensortemperature","cameratemperature","framerate","exposuretime"]
channel_units=["oC","oC",'Hz','us']
channel_values1=[sensor1, chip1,framerate1,expo1]
channel_values2=[sensor2, chip2,framerate2,expo2]
comment_meta1=""
comment_meta2=""
print(datetime.datetime.now()," regenerate timevectors")
time1=IRup.regenerate_timevector(time1,0,t2,0)
time2,Program=IRup.regenerate_timevector(time2,[1,6],t2,0,True)
time1_0=time1
time2_0=time2
hf1.close()
del attri1
hf2.close()
del attri2
metares1=0
metares2=0
rawres1=0
rawres2=0
print(datetime.datetime.now()," Upload Meta Stream(s)")
if metastream:
try:
meta1up=IRup.upload_meta_stream_V2(cameras[i],port,np.asarray(time1),channel_names,channel_units,channel_values1,newversion=newversion,reason=reason, Return=True)
if meta1up:
metares1=1
else:
metares1=0
except Exception as E:
print(E)
metares1=0
# foutf.write(datetime.datetime.now(),"metastream 1 upload failed",E,"\n")
del sensor1, chip1,framerate1,expo1
try:
meta2up=IRup.upload_meta_stream_V2(cameras[i],port,np.asarray(time2),channel_names,channel_units,channel_values2,newversion=newversion,reason=reason, Return=True)
if meta2up:
metares2=1
else:
metares2=0
except Exception as E:
print(E)
metares2=0
# foutf.write(datetime.datetime.now(),"metastream 2 upload failed",E,"\n")
del sensor2,chip2,framerate2,expo2
else:
print("metastream upload deactivated")
metares1=0
metares2=0
# foutf.write(datetime.datetime.now(),"metastream upload deactivated","\n")
if raw_parlog or raw_data:
print(datetime.datetime.now()," Start Raw Data Upload for "+str(prog_time))
if raw1:
success=False
trys=0
temp_raw=raw_parlog
while (trys<retry and not success):
try:
response,pari=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time1_0,dtype=np.int64),path+"\\"+rlist[fkeys[f1]],metadata1,parlog=temp_raw,rawdata=raw_data,newversion=newversion,reason=reason,del_list=correction_list1,temppath=temppath)#upload cold and backgrounds
if response[0]:
success=True
rawres1=1
else:
print(datetime.datetime.now(),"raw 1 upload failed",response[1],response[2])
rawres1=0
temp_raw=not pari
trys+=1
# foutf.write(datetime.datetime.now(),"raw 1 upload failed",response[1],response[2],"\n")