Commit c0ee646c authored by Holger Niemann's avatar Holger Niemann
Browse files

update to V2.9.5: bugfixes for AEFXX, correct the used dictionary for the paths

parent 48326030
......@@ -37,6 +37,12 @@ bugfixes in downloadversionIRdata:
Versions:
V3.0.0: download of heat flux data not also possible
V2.9.5: bug fixes for AEFXX, wrong dictionaries was used, use the correct defintion of the paths for the download.
V2.9.4: bug fixes for AEF50 download in OP1.2a (got trouble with the change to real Archive), fixes in the upload script to upload flir data and fix extra frames
V2.9.3: bugfixes for FLIR, ArchiveDB is now used as default as download
V2.9.2: bugfixes for AEF50 in the download
V2.9.1: bugfixes for AEF50
V2.9.0: changed multiprocessing to multithreading, better performance, less archive calls
V2.8.0: add FLIR camera into the downloadroutine, adapt the port interface, that it accept strings and numbers, numbers are automatically used for QRT
V2.7.1: prove the temperature calculation, less ram usage
V2.7.0: implement dependency on upload_config; switch between ArchiveDB and Test archive possible
......
......@@ -3,7 +3,7 @@
Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018
Version: 2.9.4
Version: 2.9.5
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes )
@author: holn
"""
......@@ -227,10 +227,10 @@ def download_NUC_by_program(port,program,exposure,version=0):
def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
project="W7X"
OP=get_OP_by_time(starttime)
larchivepath=archivepath+project+"/"+portcamdict[OP]["AEF"+str(port)]+"NUC_"
larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_"
# NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if version==0:
version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM")
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM")
try:
res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
signal_list = json.loads(res.read().decode('utf-8'))
......@@ -396,8 +396,8 @@ def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametyp
t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9)
t0date=t0date.isoformat()
if version==0:
version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
t_offset=timest0[-1][0]-time_t[0]
if t_offset>10e6:
......@@ -409,10 +409,10 @@ def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametyp
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
OP=IR_tools.get_OP_by_time(time_ns=starttime)
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_"
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_"
# NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if version==0:
version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
try:
res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9)))
signal_list = json.loads(res.read().decode('utf-8'))
......@@ -468,8 +468,8 @@ def download_background_by_program(port,program,exposure,version=0):
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
OP=IR_tools.get_OP_by_time(time_ns=starttime)
stream=portcamdict[OP]["AEF"+str(port)]+"background_"
larchivepath=archivepath+project+"/"+portcamdict[OP]["AEF"+str(port)]
stream=portpathdict[OP]["AEF"+str(port)]+"background_"
larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]
if version==0:
version=get_latest_version(stream+"DATASTREAM")
try:
......@@ -507,7 +507,7 @@ def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=
def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False):
"OP1.2b function"
exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version)
camera=portcamdict["OP1.2b"]["AEF"+str(port)]
camera=portpathdict["OP1.2b"]["AEF"+str(port)]
background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64)
for frame in backgroundframes:
background+=frame
......@@ -554,8 +554,8 @@ def download_raw_images_by_program(port,program,time_s=0,version=0,verbose=0):
def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0):
OP=IR_tools.get_OP_by_time(time_ns=starttime)
if version==0:
version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
if (stoptime-starttime)/intervalSize>1:
nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
if verbose>0:
......@@ -615,8 +615,8 @@ if fastDL:
stoptime=prog[1]['trigger']['6'][0]
OP=get_OP_by_time(starttime)
if version==0:
version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
return download_raw_images_by_time_via_png(larchivepath,starttime,stoptime,time_s=time_s,threads=threads,version=version,verbose=verbose-1)
def download_raw_images_by_time_via_png(larchivepath,starttime,stoptime,time_s=0,threads=1,version=0,verbose=0):
......@@ -827,10 +827,14 @@ def download_raw_FLIR_images_via_png(t1,t6,time_s=0,version=0,threads=1,verbose=
if mode==1:
### okay the data is stored probably as different sets (so recorded with FLIR software)
timess,versions=get_FLIR_Pset_times(stdate,enddate,version)
if verbose>0:
print("found ",np.shape(timess[0]),np.shape(timess[1]),np.shape(timess[2]),np.shape(timess[3])," frames in the different sets")
### we not know the times for all presets for the program, now we have to download them, if times is not empty
exist,time,frames,tsets=download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=verbose-1)
elif mode==2:##the data is stored only in one datastream, at least this datastream exist
raise Exception("not implemented")
#download data and downlaod LUT
# time=[]
# images=[]
# lnt=len(times)
......@@ -849,7 +853,7 @@ def download_raw_FLIR_images_via_png(t1,t6,time_s=0,version=0,threads=1,verbose=
return exist,time,frames,mode,tsets,versions
def get_FLIR_Pset_times(stdate,enddate,version=0):
def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0):
larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw"
if version==0:
versions=[]
......@@ -871,7 +875,7 @@ def get_FLIR_Pset_times(stdate,enddate,version=0):
times_0=AKF_2.get_time_intervals(larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw",stdate.replace("T"," "),enddate.replace("T"," "))#
found=True
except Exception as E:
print(E)
print(E,larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw")
found=False
times_0=[]
version0=version0-1
......@@ -884,7 +888,8 @@ def get_FLIR_Pset_times(stdate,enddate,version=0):
counter+=1
try:
times_0=AKF_2.get_time_intervals(larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw",stdate.replace("T"," "),enddate.replace("T"," "))#
except:
except Exception as E:
print(E,larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw")
times_0=[]
timess.append(times_0)
return timess,versions
......@@ -930,13 +935,14 @@ def download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=0):
larchivepaths=[]
versionss=[]
for times in timess:
times=np.array(times)
setn+=1
if times!=[]:
counter+=1
threads+=1
Psets.append(counter)
larchivepaths.append(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw")
tim.append(times)
tim.append(times[:,0])
versionss.append(versions[setn])
jobs = []
resultdict = []
......@@ -996,7 +1002,7 @@ def download_last_raw_image_by_time(larchivepath,starttime,stoptime,version=0):
pixelarray = np.array(img,dtype=np.uint16)#.swapaxes(0,1)
return True, pixelarray
except urllib.error.URLError as e:
print(e)
print(e,larchivepath+"/_signal.png?from="+str(starttime-10)+"&upto="+str(stoptime))
return False, [-1]
def download_raw_parlog_by_program(port,program,version=0):
......@@ -1015,8 +1021,8 @@ def download_raw_parlog_by_program(port,program,version=0):
def download_raw_parlog_by_times(port,starttime,stoptime,version=0):
OP=IR_tools.get_OP_by_time(time_ns=stoptime)
if version==0:
version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_PARLOG")
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version)
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG")
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version)
try:
res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
signal_list = json.loads(res.read().decode('utf-8'))
......@@ -1045,8 +1051,8 @@ def get_INFRATEC_filter_by_program(program,version=0):
def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0):
OP=IR_tools.get_OP_by_time(time_ns=starttime)
if version==0:
version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_PARLOG")
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version)
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG")
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version)
try:
res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
signal_list = json.loads(res.read().decode('utf-8'))
......@@ -1076,8 +1082,8 @@ def get_exposure_by_program(port,program,version=0):
def get_exposure_by_times(port,starttime,stoptime,version=0):
OP=IR_tools.get_OP_by_time(time_ns=stoptime)
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"meta_"
stream=portcamdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/3/exposuretime"+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
......@@ -1097,8 +1103,8 @@ def get_camera_temp_by_program(port,program,version=0):
def get_camera_temp_by_times(port,starttime,stoptime,version=0):
OP=IR_tools.get_OP_by_time(time_ns=stoptime)
stream=portcamdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"meta_"
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
dummy=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
......@@ -1122,8 +1128,8 @@ def get_sensor_temp_by_program(port,program,version=0):
def get_sensor_temp_by_times(port,starttime,stoptime,version=0):
OP=IR_tools.get_OP_by_time(time_ns=stoptime)
stream=portcamdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"meta_"
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
......@@ -1144,8 +1150,8 @@ def get_camera_framerate_by_program(port,program,version=0):
def get_camera_framerate_by_times(port,starttime,stoptime,version=0):
OP=IR_tools.get_OP_by_time(time_ns=stoptime)
stream=portcamdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"meta_"
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
......@@ -1166,8 +1172,8 @@ def get_frametype_by_program(port,program,version=0):
def get_frametype_by_times(port,starttime,stoptime,version=0):
OP=IR_tools.get_OP_by_time(time_ns=starttime)
stream=portcamdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"meta_"
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/5/frametype/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
......@@ -1187,8 +1193,8 @@ def get_framenumber_by_program(port,program,version=0):
def get_framenumber_by_times(port,starttime,stoptime,version=0):
OP=IR_tools.get_OP_by_time(time_ns=starttime)
stream=portcamdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"meta_"
stream=portpathdict[OP]["AEF"+str(port)]+"meta_"
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/4/framenumber/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
......@@ -1572,8 +1578,8 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
return False,[0],[0],False
else:
if version==0:
version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
if verbose>0:
print(datetime.datetime.now(),"raw download start")
if fastDL:
......@@ -1651,8 +1657,8 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_v
success=True
OP=get_OP_by_time(starttime)
if version==0:
version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
if type(time_s)==list:
tstart=time_s[0]
tstop=time_s[1]
......@@ -1772,8 +1778,8 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_versi
success=True
OP=get_OP_by_time(t1)
if version==0:
version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM")
larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw"
if type(time_s)==list:
tstart=time_s[0]
tstop=time_s[1]
......@@ -2355,9 +2361,9 @@ end of the copied functions
if __name__=='__main__':
print("local function calling")
port="AEF51"
prog="20180925.065"
status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,4],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
port=50#"AEF51"
prog="20180925.017"
status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,0.15],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False,verbose=5)
if status:
plt.figure()
plt.imshow(images[-1],vmin=330,vmax=1000,cmap=exJet)
......
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 24 11:24:52 2017
Version: 2.9.4
Version: 2.9.5
@author: holn
"""
......@@ -643,7 +643,7 @@ def upload_NUC_ref(camera,portnum,time,texp_filter,gain,offset,cold,bad,gain_e=[
upload_fulldataset(url,parobj,dataobj,version,parlog,data)
def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason="",extra_par=None):
def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,group="QRT_IRCAM",newversion=False,reason="",extra_par=None):
"""
upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason="")
(numpy.array) LUTs: t x n x 3 array DL,T,T_er, in general t=1, n= number of look-up-tables
......@@ -655,10 +655,15 @@ def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason=""
(str) reason: reason for the new version
"""
#camera="IRCAM" #will be removed in final version
group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
# group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
name=["LUT"]
project="W7X"
datastream_name="AEF"+str(portnum)+"_LUT"
if type(portnum)==int:
datastream_name="AEF"+str(portnum)+"_LUT"
descri="LUT for "+camera+" in AEF"+str(portnum)
elif type(portnum)==str:
datastream_name=portnum+"_LUT"
descri="LUT for "+camera+" in "+portnum
#test of dataformat
if np.shape(LUTs)[0]!=len(time):
print("missmatch in between time vector and LUTs timestamps")
......@@ -678,6 +683,8 @@ def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason=""
for n in range(np.shape(LUTref[0])[0]):
if camera=="INFRATEC":
structure["Filter_"+str(LUTref[0][n][0])+"_Texp_"+str(LUTref[0][n][1])+"us_e_"+str(LUTref[0][n][2])]=2*n+1
elif camera=="FLIR":
structure["Texp_"+str(LUTref[0][n][0])+"us_e_"+str(LUTref[0][n][1])]=2*n+1
else:
structure["Texp_"+str(LUTref[0][n][0])+"us_e_"+str(LUTref[0][n][1])]=2*n+1
else:
......@@ -701,7 +708,7 @@ def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason=""
#using here selfmade uploadroutine to avoid that LUTs table appear as images
chanDesc={"[0]":{'name':name[0],'physicalQuantity':{'type':'none'},'active':1}}
parobj={"label":"parms",
"description":"LUT for "+camera+" in AEF"+str(portnum),
"description":descri,
"values":[{"chanDescs":chanDesc,"structure":structure}],
"dimensions":[time[0],time[0]+10]
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment