diff --git a/CHANGELOG b/CHANGELOG
index 330ec075046536edc1b439c0c8fed7fcda40d686..828b1cf6819623570cb2324b5358287aca89a1e3 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,9 @@
+21.08.2018:
+update downloadversionIRdata:
+ - include version number, now 2.5.3 into the files
+ - bug fixes for downloadversion IRdata
+ - improvement of exception handling, replace some exception with warnings and improve error messages
+
03.11.2017:
changes in downloadversionIRdata:
- reduce blocks to 1 second, now editable over parameter intervalSize
diff --git a/IR_image_tools.py b/IR_image_tools.py
index f9dd56c36d3fc33142c6a06f79882b637f12bbf7..9df7318d4eab758ee4ff7b050bd6487d4b8719c2 100644
--- a/IR_image_tools.py
+++ b/IR_image_tools.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
Created on Wed May 9 14:56:32 2018
-
+Version: 2.5.3
@author: Holger Niemann, Peter Drewelow, Yu Gao
mainly to clean up the downloadversionIRdata code
diff --git a/Upload_Day_IRdata.py b/Upload_Day_IRdata.py
index 9ddc92b3458f2902f8958693a76d508a272d31b6..8a256cb1eed2f405dbde95d37b470465c9be44fe 100644
--- a/Upload_Day_IRdata.py
+++ b/Upload_Day_IRdata.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 26 16:01:06 2017
-
+Version: 2.5.3
@author: holn
"""
@@ -514,15 +514,17 @@ try:
if raw1:
success=False
trys=0
+ temp_raw=raw_parlog
while (trys<retry and not success):
try:
- response=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time1_0,dtype=np.int64),path+"\\"+rlist[fkeys[f1]],metadata1,parlog=raw_parlog,rawdata=raw_data,newversion=newversion,reason=reason,del_list=correction_list1,temppath=temppath)#upload cold and backgrounds
+ response,pari=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time1_0,dtype=np.int64),path+"\\"+rlist[fkeys[f1]],metadata1,parlog=temp_raw,rawdata=raw_data,newversion=newversion,reason=reason,del_list=correction_list1,temppath=temppath)#upload cold and backgrounds
if response[0]:
success=True
rawres1=1
else:
print(datetime.datetime.now(),"raw 1 upload failed",response[1],response[2])
rawres1=0
+ temp_raw=not pari
# foutf.write(datetime.datetime.now(),"raw 1 upload failed",response[1],response[2],"\n")
except Exception as E:
trys+=1
@@ -535,16 +537,18 @@ try:
if raw2:
success=False
trys=0
+ temp_raw=raw_parlog
while (trys<retry and not success):
print(datetime.datetime.now()," Upload second file")
try:
- response=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time2_0,dtype=np.int64),path+"\\"+rlist[fkeys[f2]],metadata2,parlog=raw_parlog,rawdata=raw_data,del_list=correction_list2,temppath=temppath)#upload raw images from plasma
+ response,pari=IRup.upload_raw_data_from_H5(cameras[i],port,np.asarray(time2_0,dtype=np.int64),path+"\\"+rlist[fkeys[f2]],metadata2,parlog=temp_raw,rawdata=raw_data,del_list=correction_list2,temppath=temppath)#upload raw images from plasma
if response[0]:
success=True
rawres2=1
else:
print(datetime.datetime.now(),"raw 2 upload failed",response[1],response[2])
rawres2=0
+ temp_raw=not pari
# foutf.write(datetime.datetime.now(),"raw 2 upload failed",response[1],response[2],"\n")
except Exception as E:
trys+=1
diff --git a/downloadversionIRdata.py b/downloadversionIRdata.py
index 599cdc6516a5a9e9c7c81f2287b4de864c55699a..fb6e54cf260dc6a9954eafdb79089ee017bb34be 100644
--- a/downloadversionIRdata.py
+++ b/downloadversionIRdata.py
@@ -1,7 +1,10 @@
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 25 15:51:46 2017
+updated on Tue Aug 21 10:20:00 2018
+Version: 2.5.3
+(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2) . #of updates to add functionalities . #number of updates for bug fixes )
@author: holn
"""
@@ -278,9 +281,12 @@ def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
OP=IR_tools.get_OP_by_time(time_ns=endtime)
if OP=="OP1.2a":
+ t1=endtime
+ t0=starttime
+ prog=read_program(t1)
+ program=prog[1]['id']
if (port == 31) or (port == 21 and float(program[4:]) > 1110):
- t1=endtime
- t0=starttime
+
print("rebuilding coldframe")
#use any way the rebuilt coldframe.
sT = get_sensor_temp_by_program(port, program)[2][0]
@@ -321,7 +327,7 @@ def get_NUC_by_times(port,starttime,endtime,t_exp,version=0,verbose=0):
badpixels=find_badpixels(port,gain,offset,niterations=10,tolerance=10)
gain_error=0
offset_error=0
- return True,[gain,offset,cirebuild,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
+ return True,[gain,offset,ci,badpixels,gain_error,offset_error],['gain','offset','cold','badpixels','gain_error','offset_error']
else:
return download_NUC_by_times(port,starttime,endtime,t_exp)
elif OP=="OP1.2b":
@@ -615,7 +621,11 @@ if fastDL:
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
- if prog[0]:
+ if not prog[0]:
+ if verbose>0:
+ print("program not found, cannot download the data")
+ return False,[0],[0]
+ else:
starttime=prog[1]['trigger']['1'][0]
stoptime=prog[1]['trigger']['6'][0]
success=True
@@ -655,7 +665,11 @@ if fastDL:
enddate=datetime.datetime.utcfromtimestamp((starttime)/1e9+time_s)
enddate=enddate.isoformat()
#"2017-11-15 08:00:00"
- times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
+ try:
+ times=AKF_2.get_time_intervals(larchivepath,stdate.replace("T"," "),enddate.replace("T"," "))#
+ except Exception as E:
+ raise Warning(E)
+ return False,[0],[0]
time=[]
images=[]
lnt=len(times)
@@ -739,9 +753,9 @@ if fastDL:
else:
successes.append(False)
if give_ERROR:
- images,error_images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
+ sucess,images,error_images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
else:
- images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
+ success,images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR)
for i in range(len(images)):
images[i]=(images[i]*FOV).astype(np.float32)
@@ -794,7 +808,7 @@ def download_last_raw_image_by_time(port,starttime,stoptime,version=0):
return True, pixelarray
except urllib.error.URLError as e:
print(e)
- return False, -1
+ return False, [-1]
def download_raw_parlog_by_program(port,program,version=0):
# prog=AKF_1.get_program_from_PID(program)
@@ -1231,7 +1245,7 @@ def get_temp_from_raw_by_program_V1(portnr,program,time_s=0,emi=0.8,divertorpart
del raw_dl
# images=images.swapaxes(1,2)
images=[im.swapaxes(0,1) for im in images]
- images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,True)
+ sucess,images=apply_calib_on_raw(images,background,LUT,refT,gain,offset,True)
temperatureimages=temperatureimages+images
times=times+time
del time,images
@@ -1246,9 +1260,9 @@ def get_temp_from_raw_by_program_V1(portnr,program,time_s=0,emi=0.8,divertorpart
# temperatureimages=temperatureimages.swapaxes(1,2)
temperatureimages=[im.swapaxes(0,1) for im in temperatureimages]
if give_ERROR:
- temperatureimages,error_images=apply_calib_on_raw(temperatureimages,background,LUT,refT,gain,offset,gain_error,offset_error,True,give_ERROR)
+ success,temperatureimages,error_images=apply_calib_on_raw(temperatureimages,background,LUT,refT,gain,offset,gain_error,offset_error,True,give_ERROR)
else:
- temperatureimages=apply_calib_on_raw(temperatureimages,background,LUT,refT,gain,offset,gain_error,offset_error,True,give_ERROR)
+ success,temperatureimages=apply_calib_on_raw(temperatureimages,background,LUT,refT,gain,offset,gain_error,offset_error,True,give_ERROR)
else:
raise Exception("cannot download the raw images")
valid=True
@@ -1271,56 +1285,71 @@ def get_temp_from_raw_by_program_V2(portnr,program,time_s=0,emi=0.8,version=0,th
return get_temp_from_raw_by_program(portnr,program,time_s=time_s,emi=emi,T_version=2,version=version,threads=threads,give_ERROR=give_ERROR,use_firstframe_as_background=use_firstframe_as_background)
def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,back_emissivity=0.8,verbose=0):
- background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version,back_emissivity,verbose=verbose-1)
- if verbose>0:
- print(datetime.datetime.now(),"raw download start")
- if fastDL:
- exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_s,version,threads,verbose=verbose-1)
+ cexist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version,back_emissivity,verbose=verbose-1)
+ if not cexist:
+ if verbose>0:
+ print("Unable to load the complete calibration data for",program," please confirm that this data was uploaded")
+ return False,[0],[0],False
else:
- exist,time,frames=download_raw_images_by_program(portnr,program,time_s,version,verbose=verbose-1)
- if exist:
- frames=[im.swapaxes(0,1) for im in frames]
+ if verbose>0:
+ print(datetime.datetime.now(),"raw download start")
+ if fastDL:
+ exist,time,frames=download_raw_images_by_program_via_png(portnr,program,time_s,version,threads,verbose=verbose-1)
else:
- raise Exception("no data found")
- if verbose>0:
- print(datetime.datetime.now(),"raw download finished")
- FOV=get_FOV_mask(portnr)
- if use_firstframe_as_background:
- bim=apply_NUC([frames[0]],gain,offset)
- background=get_average_background_recangle(portnr,bim[0])
- if give_ERROR:
- frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
- else:
- frames=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
- for i in range(len(frames)):
- frames[i]=(frames[i]*FOV).astype(np.float32)
- if verbose>0:
- print(datetime.datetime.now(),"correcting bad pixels")
- frames=IR_tools.correct_images(frames,badpixels)
- if verbose>0:
- print(datetime.datetime.now(),"checking temperaturerange")
- valid=True
-# for i in range(len(frames)):
- try:
- check_temperature_range(time[0],np.max(frames)-273.15,np.min(frames[:][np.nonzero(frames[i])])-273.15,portnr,t_exp,cfilter)
- except Exception as E:
- print(E)
- valid=False
-# i=len(frames)
- date=int(program.split(".")[0])
- if date>20180712 and date<20180720:
- prog=AKF_1.get_program_from_PID(program)
- if prog[0]:
-# t0time=prog[1]['trigger']['0'][0]
- starttime=prog[1]['trigger']['1'][0]
- toff=time[0]-starttime
- if toff>10e6:
- print("time offset detected, offset is "+str(toff)+" ns, correcting")
- time[:]=time[:]-toff
- if give_ERROR:
- return exist,time,frames,valid,error_images
- else:
- return exist,time,frames,valid
+ exist,time,frames=download_raw_images_by_program(portnr,program,time_s,version,verbose=verbose-1)
+ if exist:
+ frames=[im.swapaxes(0,1) for im in frames]
+ else:
+ raise Exception("no data found")
+ if not exist:
+ raise Warning("data not found in database!")
+ return False,[0],[0],False
+ else:
+ if verbose>0:
+ print(datetime.datetime.now(),"raw download finished")
+ FOV=get_FOV_mask(portnr)
+ if use_firstframe_as_background:
+ bim=apply_NUC([frames[0]],gain,offset)
+ background=get_average_background_recangle(portnr,bim[0])
+ if give_ERROR:
+ success,frames,error_images=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
+ else:
+ success,frames=apply_calib_on_raw(frames,background,LUT,refT,gain,offset,gain_error,offset_error,False,give_ERROR,verbose=verbose-1)
+ if not success:
+ if verbose>0:
+ print("calculation of the temperature failed, calibration could not be applied")
+ return False,time,frames,False
+ else:
+
+ for i in range(len(frames)):
+ frames[i]=(frames[i]*FOV).astype(np.float32)
+ if verbose>0:
+ print(datetime.datetime.now(),"correcting bad pixels")
+ frames=IR_tools.correct_images(frames,badpixels)
+ if verbose>0:
+ print(datetime.datetime.now(),"checking temperaturerange")
+ valid=True
+ # for i in range(len(frames)):
+ try:
+ check_temperature_range(time[0],np.max(frames)-273.15,np.min(frames[:][np.nonzero(frames[i])])-273.15,portnr,t_exp,cfilter)
+ except Exception as E:
+ print(E)
+ valid=False
+ # i=len(frames)
+ date=int(program.split(".")[0])
+ if date>20180712 and date<20180720:
+ prog=AKF_1.get_program_from_PID(program)
+ if prog[0]:
+ # t0time=prog[1]['trigger']['0'][0]
+ starttime=prog[1]['trigger']['1'][0]
+ toff=time[0]-starttime
+ if toff>10e6:
+ print("time offset detected, offset is "+str(toff)+" ns, correcting")
+ time[:]=time[:]-toff
+ if give_ERROR:
+ return exist,time,frames,valid,error_images
+ else:
+ return exist,time,frames,valid
def get_temp_from_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_version=2,version=0,threads=1,give_ERROR=False,check_range=True):
if threads==1 or not fastDL:
@@ -1542,6 +1571,10 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_versi
return success,np.array(times),images,valid
def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_error=0,offset_error=0,fullbackground=False,give_ERROR=False, verbose=0):
+ """
+ apply_calib_on_raw
+ return success,images and if give_Error: errorimages
+ """
try:
#images=np.array(raw,dtype=np.uint16)
# del raw
@@ -1584,15 +1617,15 @@ def apply_calib_on_raw(images,background,LUT,refT=28.5,gain=0,offset=0,gain_erro
terror[i] = (terror[i] + (max_image - min_image)/2).astype(np.float32)
images[i]=(temp_from_LUT(LUT,images[i])).astype(np.float32)
images[i]=(images[i]+(refT+273.15)).astype(np.float32)
- return images,terror
+ return True,images,terror
else:
for i in range(len(images)):
images[i]=(temp_from_LUT(LUT,images[i])).astype(np.float32)
images[i]=(images[i]+(refT+273.15)).astype(np.float32)
- return images
+ return True,images
except Exception as E:
- print(E)
- return 0
+ raise Warning(E)
+ return False,[0]
def apply_NUC(images,gain,offset,gain_error=0,offset_error=0,give_error=False):
"""
@@ -1647,8 +1680,13 @@ def get_cooling_water_temperature_by_time(time_ns):
return result_in[0] and result_out[0],inlet,outlet
def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivity=0.82, verbose=0):
+ """
+ get_calib_data
+ return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
+ """
prog=AKF_1.get_program_from_PID(program)
OP=IR_tools.get_OP_by_time(program_str=program)
+ exist=True
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
@@ -1659,7 +1697,9 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
# except Exception as E:
# print("Warning: Program was not found: ",E)
# return 0,0,0,0,0,0,0,0,0,0
-# OP=get_OP_by_program(program)
+ else:
+ raise Warning("cannot find the program!")
+ return False,0,0,0,0,0,0,0,0,0,0
Camera=portcamdict[OP]['AEF'+str(port)]
if Camera.split("_")[0]=="Infratec":
cf=get_INFRATEC_filter_by_program(program,version)
@@ -1667,7 +1707,8 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
cfilter=cf[2]
else:
cfilter=-1
- raise Exception("Filter not found for INFRATEC Camera")
+ raise Warning("Filter not found for INFRATEC Camera for Program "+prog)
+ return False,0,0,0,0,0,0,0,0,0,0
else:
cfilter=0
expo_DL=get_exposure_by_times(port,t1,t6,version)
@@ -1682,7 +1723,8 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
if cfilter==0 and not Camera.split("_")[0]=="Infratec":
NUC_DL=get_NUC_by_times(port,t0,t1,t_exp,version,verbose=verbose-1)#download_NUC_by_times(port,t0,t1,t_exp,version)
if NUC_DL[0]==False:
- raise Exception("NUC was not found")
+ raise Warning("NUC was not found")
+ return False,0,LUT,0,0,0,0,t_exp,cfilter,0,0
else:
gain=np.array(NUC_DL[1][0])
offset=np.array(NUC_DL[1][1])
@@ -1697,7 +1739,8 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
bim=apply_NUC([frames[0]],gain,offset)
background=get_average_background_recangle(port,bim[0])
else:
- raise Exception("cannot find the first frame of the discharge, reconstruction failed")
+ raise Warning("cannot find the first frame of the discharge, reconstruction failed")
+ return False,0,LUT,0,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
else:
gain=0
offset=0
@@ -1763,14 +1806,18 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
# back_off=estimate_offset(port,program)
# background=(background-back_off)/(back_emissivity)+back_off
else:
- raise Exception("Unknown temperature calibration method")
+ raise Warning("Unknown temperature calibration method")
+ return False,background,LUT,0,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
else:
- raise Exception("no background image found")
+ raise Warning("no background image found")
+ return False,0,LUT,0,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
else:
- raise Exception("no LUT found")
+ raise Warning("no LUT found")
+ return False,0,0,0,0,0,0,t_exp,cfilter,0,0
else:
- raise Exception("no exposure time found")
+ raise Warning("no exposure time found")
+ return False,0,0,0,0,0,0,0,cfilter,0,0
if np.max(badpixels)==0:
if Camera.split("_")[0]=="Infratec":
badpixels=find_badpixels(port,frame-background,offset,niterations=10,tolerance=10,plot_it=False)#find_badpixels(port,gain,offset)
@@ -1781,7 +1828,7 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
init_bp_list = IR_tools.read_bad_pixels_from_file(port, program=program)
# find more bad pixel
badpixels = find_badpixels(port, gain, offset, init_bp_list=init_bp_list, niterations=10, tolerance=10, verbose=verbose-1)
- return background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
+ return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
# except Exception as E:
# print("Warning: Program was not found: ",E)
# return 0,0,0,0,0,0,0,0,0,0
@@ -1949,7 +1996,10 @@ if __name__=='__main__':
plt.plot(framerate[1],framerate[2],label="AEF"+str(port))
plt.legend()
plt.grid()
-# status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,6],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
+ port=10
+ prog="20180816.016"
+ prog="20180816.018"#[0,6]
+ status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=0,emi=0.82,T_version=2,version=0,threads=1,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
# status,time,images=download_raw_images_by_program_via_png(port,prog,time_s=1,threads=4,verbose=0)
# test=get_NUC_by_program(port,prog,8,version=0,verbose=5)
diff --git a/uploadingversionIRdata.py b/uploadingversionIRdata.py
index 4467d3b9dba7380a3dfe654d955cf7b32cfad4df..e739606c9b42157c8034512cec66a7a282c67780 100644
--- a/uploadingversionIRdata.py
+++ b/uploadingversionIRdata.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 24 11:24:52 2017
-
-@author: aali
+Version: 2.5.3
+@author: holn
"""
### code follows ###
@@ -364,6 +364,10 @@ def upload_meta_stream_V2(camera,portnum,time,channel_names,channel_units,channe
}
if Return: return upload_fulldataset(url,parobj,dataobj,version,Return=Return)
else: upload_fulldataset(url,parobj,dataobj,version)
+ else:
+ print("Length of timevector and length of channel values does not match or the number of channel names does not match the number of units")
+ if Return:
+ return False
def upload_raw_data(camera,portnum,time,images,metadata={},comment="",newversion=False,reason=""):
"""
@@ -470,7 +474,9 @@ def upload_raw_data_from_H5(camera,portnum,time,h5file,metadata={},comment="",pa
}
if parlog:
print("writing parlog")
- upload_Parlog(url,parobj,version)
+ parlog_success=upload_Parlog(url,parobj,version,Return=False)
+ else:
+ parlog_success=True
if rawdata:
print("writing done, creating upload H5file")
tmp=writeH5_from_File(datastream_name,h5file,'images',time,double_list=del_list,path=temppath)
@@ -487,9 +493,9 @@ def upload_raw_data_from_H5(camera,portnum,time,h5file,metadata={},comment="",pa
else:
success=False
print("upload of part "+str(n-1)+" of "+str(len(tmp))+" failed")
- return [success]
+ return [success],parlog_success
else:
- return uploadH5_Versioning(group,datastream_name,tmp,version,True)
+ return uploadH5_Versioning(group,datastream_name,tmp,version,True),parlog_success
def upload_temperaure_data(camera,portnum,images,time,metadata={},comment="",newversion=False,reason=""):
"""
@@ -851,10 +857,16 @@ def upload_Parlog(url,parobj,versionnumber=0,Return=False):
try:
parlog_response = urllib.request.urlopen(parlog_request)
if Return:
- return parlog_response.getcode()//100 is 2
+ if parlog_response.getcode()//100 is 2:
+ return True
+ else:
+ print(parlog_response.getcode())
+ return False
except urllib.error.HTTPError as ex:
msg = ex.read()
- if Return: False
+ if Return:
+ print(msg)
+ return False
else:
raise RuntimeError(msg)
@@ -886,27 +898,39 @@ def upload_fulldataset(url,parlog,data,versionnumber,bparlog=True,bdata=True,Ret
### from here on subfunctions
-def upload_images(camera,portnum,images,time,typ,newversion=False,reason=""):
+def upload_images(images,time,group,stream,newversion=False,reason=""):
"""
upload_images(camera,portnum,images,time,metadata={},newversion=False,reason="")
"""
#camera="IRCAM" #will be removed in final version
- group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
+ group=group#"QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
# names=[typ]
- datastream_name="AEF"+str(portnum)+"_"+typ#+"_TEST001"
-
+ datastream_name=stream#"AEF"+str(portnum)+"_"+typ#+"_TEST001"
#url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
- if newversion:
- print('no version control of HDF5 files yet!')
-# archive_create_stream_version(url, reason, False, "n/a", "n/a" )
+ url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
+ if newversion:
+ archive_create_stream_version(url, reason, False, "n/a", "n/a" )
+ ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
+ version=ver
+ else:
+ ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
+ if ver==None:
+ reason="Versioning of Data"
+ archive_create_stream_version(url, reason, False, "n/a", "n/a" )
+ version=1
+ else:
+ version=ver
try:
# archive.write_signals(images,time,group,project,names, datastream_name,units,metadata,2,True)
tmp=writeH5(datastream_name,images,time)
- uploadH5(group,datastream_name,tmp,True)
+# uploadH5(group,datastream_name,tmp,True)
+ response=uploadH5_Versioning(group,datastream_name,tmp,version,True)
+ return response
except Exception as E:
print("Upload Failed")
print(E)
+ return [False]
def uploadH5(group,stream, h5file, delete=False, timeout=None):
# path=Path, h5file=h5-file