Commit ed1518ff authored by Holger Niemann's avatar Holger Niemann
Browse files

update V3.0.0: heat flux up and download

parent 1c90faa0
<<<<<<< HEAD
27/28/29.11.2018: update to V3
- add file upload_heatflux.py
- add plot_heatflux_example.py
update downloadversionIRdata:
- add functions download_heatflux_by_program,download_heatflux_by_times, download_heatflux_profile, extract_heatflux_profile_from_DL
- rename download_raw_images_by_time_via_png into download_images_by_time_via_png
update uploadversionIRdata:
- add function write_H5_images, a new less ram intense version of write_H5 function for better image upload (specially heatflux)
26.09.2018:
update downloadversionIRdata:
- include FLIR into the downloadscript, Case1 (V2.8.0)
......@@ -18,7 +26,6 @@ update downloadversionIRdata:
- include version number, now 2.5.3 into the files
- bug fixes for downloadversion IRdata
- improvement of exception handling, replace some exception with warnings and improve error messages
=======
01.08.2017:
changes in downloadversionIRdata:
......@@ -36,7 +43,9 @@ bugfixes in downloadversionIRdata:
- fixed: wrong time intervall for TC divertor data
Versions:
V3.0.0: download of heat flux data not also possible
V3.0.0: download and upload of heat flux data now also possible, the script creates a request+memory file for all heatflux request which point to a camera, program comibination which was not uploaded; in this way high requested heatflux data can be evaluated earlier if possible
V2.9.7: bug fixes in the download for OP1.2a data, background download was broken
V2.9.6: bug fixes for AEFXX, port and portnr was not used in the right way
V2.9.5: bug fixes for AEFXX, wrong dictionaries was used, use the correct defintion of the paths for the download.
V2.9.4: bug fixes for AEF50 download in OP1.2a (got trouble with the change to real Archive), fixes in the upload script to upload flir data and fix extra frames
V2.9.3: bugfixes for FLIR, ArchiveDB is now used as default as download
......
# -*- coding: utf-8 -*-
"""
Created on Tue Jun 5 09:26:06 2018
Version: 2.9.4
Version: 3.0.0
@author: holn
constants and config parameters
"""
......@@ -10,6 +10,7 @@ calibpath="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Calibrations\\"
IRCamColdframes_fittingpath=calibpath+"IRCamColdframes_fitting\\"
IRCamRefImagespath = calibpath+'IRCamReferenceImages\\'
IRCAMBadPixels_path = calibpath + 'IRCAMBadPixels\\'
heatflux_requestlist_path="\\\\x-drive\\Diagnostic-logbooks\\QRT-DivertorThermography\\ArchiveDB_heatflux_pipeline\\"
try:
# import sys
path=str(repr(__file__)).split("IR_config_constants")[0].split("'")[1]+"upload_config"
......
This diff is collapsed.
# -*- coding: utf-8 -*-
"""
Created on Thu Nov 29 17:41:40 2018
V3.0.0
@author: holn
"""
import numpy as np
import downloadversionIRdata as IR
import matplotlib.pyplot as plt
import datetime
if __name__=='__main__':
#%% loading data
print(datetime.datetime.now(),"start")
status,times,images=IR.download_heatflux_by_program(20,"20171109.008",timewindow=3.1,version=1,threads=1,verbose=5)
print('done')
#%% plotting data
status2,time,s,q=IR.extract_heatflux_profile_from_DL((np.asarray(times)-times[0])/1e9,images,profile="TM3h_5_5",timepoint=1,verbose=5)
plt.figure()
plt.imshow(images[220]/1e6)
plt.clim([0,4])
plt.tick_params(axis='both', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
left='off',
right='off',
labelleft='off',
labelbottom='off') # labels along the bottom edge are off
c_ax = plt.colorbar()
c_ax.set_label('q [MW/m2]')
plt.savefig('20171109.008 - 2.2s - AEF20IR - in MWm2.png', dpi=300, bbox_inches='tight')
plt.figure()
plt.plot(s,q/1e6,'ko-')
plt.xlabel("s [m]",fontsize=26)
plt.ylabel("q [MW/m2]",fontsize=26)
plt.tick_params(labelsize=26)
plt.title("20171109.008 heat flux profile for TM3, finger 5, profile 5 @1s",fontsize=26)
plt.grid()
plt.ylim(bottom=0)
plt.show()
\ No newline at end of file
# -*- coding: utf-8 -*-
"""
Created on Fri Sep 21 09:32:29 2018
@author: holn
"""
import numpy as np
import scipy.io as sio
import matplotlib.pyplot as plt
import h5py as h5
import uploadingversionIRdata as IRup
import downloadversionIRdata as IR
import datetime
#125,190 vertical, horizontal *3 mm
port=20
program="20171109.008"
path="F:\\holn\\Documents\\Doktorarbeit\\RESULTS\\OP1.2a\\Fulldivertor_res\\EJM\\He\\"
outpfad="F:\\holn\\Documents\\Doktorarbeit\\DATEN\\Mapping\\LinesforStacks\\"
project="W7XAnalysis"
group="QRT_IRCAM"
stream="AEF"+str(port)+"_heatflux"
comment="Test upload"
header={'code_version':4,'alpha_corrected':1}
stream2="Mapping_reference"
upload_Mapping=True
File=sio.loadmat(path+program+"_AEF"+str(port)+"_heatflux_V4.mat")
time=np.asarray(File['time'][0])
tilelist=np.asarray(File['tiles'])
heatpics=np.zeros((125+190+10,108*12+4,len(time)))
alphapic=np.zeros((125+190+10,108*12+4,1))
alphapic.fill(np.nan)
heatpics.fill(np.nan)
Xpic=alphapic.copy()
Ypic=alphapic.copy()
FXpic=alphapic.copy()
FYpic=alphapic.copy()
profpic=alphapic.copy()
profnpic=alphapic.copy()
tilepic=alphapic.copy()
fingerpic=alphapic.copy()
linedict={'h_l':30,'h_m':54,'h_h':24,'v':29}
numstacks={
'h_l':{0:13,1:14,2:14,3:14,4:14,5:14,6:14,7:14,8:14,9:14,10:15,11:15,12:14,13:14,14:14,15:13,16:13,17:13,18:13,19:13,20:13,21:13,22:13,23:13,24:13,25:13,26:13,27:13,28:13,29:13},
'h_m':{0:14,1:15,2:14,3:15,4:14,5:15,6:14,7:15,8:14,9:15,10:14,11:15,12:14,13:15,14:14,15:15,16:14,17:15,18:14,19:15,20:14,21:15,22:14,23:15,24:14,25:15,26:14,27:15,28:14,29:15,30:14,31:15,32:14,33:15,34:14,35:15,36:14,37:15,38:14,39:15,40:14,41:15,42:14,43:15,44:14,45:15,46:14,47:15,48:15,49:14,50:14,51:14,52:14,53:15},
'h_h':{0:16,1:15,2:15,3:15,4:15,5:15,6:15,7:15,8:15,9:15,10:15,11:15,12:15,13:15,14:15,15:15,16:15,17:15,18:15,19:15,20:15,21:15,22:15,23:15},
'v':{0:15,1:15,2:15,3:15,4:15,5:15,6:15,7:15,8:15,9:15,10:15,11:15,12:15,13:15,14:15,15:15,16:15,17:15,18:15,19:15,20:15,21:15,22:15,23:15,24:15,25:15,26:15,27:15,28:15}
}
def splitStacktoLines(locationall, rowcountarray):
locations = []
for i in range(len(rowcountarray) - 1):
locations.append(np.array(locationall[rowcountarray[i]:rowcountarray[i+1]]))
locations = np.array(locations)
return locations
ii=0
V_index=0
xoff=0
maxyoff=0
tn=0
for tile in tilelist:#['h_l','h_m','h_h','v']:
tn+=1
tile=tile.replace(" ","")
linn=linedict[tile]
if tile!='v':
V_index+=linn
yoff=130
else:
yoff=1
xoff=1
for i in range(linn):
if ii<30:
iko=ii
elif ii<84:
iko=ii-30
elif ii<108:
iko=ii-84
else:
iko=ii-108
if ii>=78 and ii<108:
tn=3
fingerfile=str(port//10)
if port % 10 ==0:
fingerfile=fingerfile+"l"+tile+"_"
else:
fingerfile=fingerfile+"u"+tile+"_"
if port==20 and int(program.split(".")[0])<20171020:
linefile=fingerfile+'{0:02d}'.format(iko)+"2Dlines_biliniar_before_20171020.h5"
else:
linefile=fingerfile+'{0:02d}'.format(iko)+"2Dlines_biliniar.h5"
linf=h5.File(outpfad+linefile,'r')
linid=list(linf['LineNos'])
lines=linf['locationall2D']#
linlocid=linf['rowcountarray']
locations=splitStacktoLines(lines,linlocid)
linf.close()
xoff+=1
if tile=='h_m' and iko<48 and iko%2==1:
shift=True
xoff=xoff-numstacks[tile][i-1]-1
maxyoff+=1
else:
shift=False
maxyoff=0
for j in range(numstacks[tile][i]):
j2=linid.index(j)
if shift:
yloff=maxyoff
else:
yloff=0
lprof=np.array(File['profiles_'+str(ii)+"_"+str(j2)][0])
lalpha=np.array(File['alpha_'+str(ii)+"_"+str(j2)][0])
lheat=np.array(File['heat_'+str(ii)+"_"+str(j2)])
lXY=np.array(File['plot_profiles_'+str(ii)+"_"+str(j2)])
loco=locations[j2]
profilelength=int(np.max(lprof)/3e-3)+1
for k in range(profilelength):
pos=k*3e-3#3.143e-3
pid=IR.IR_tools.find_nearest(lprof,pos)
if abs(pos-lprof[pid])<2e-3:
alphapic[yoff+yloff,xoff]=lalpha[pid]
heatpics[yoff+yloff,xoff,:]=lheat[:,pid]
Xpic[yoff+yloff,xoff]=lXY[pid,0]
Ypic[yoff+yloff,xoff]=lXY[pid,1]
FXpic[yoff+yloff,xoff]=loco[pid,0]
FYpic[yoff+yloff,xoff]=loco[pid,1]
profpic[yoff+yloff,xoff]=lprof[pid]
# profnpic[yoff+yloff,xoff]=j
tilepic[yoff+yloff,xoff]=tn
fingerpic[yoff+yloff,xoff]=ii*100+j
yloff+=1
xoff+=1
if yloff>maxyoff and not shift:
maxyoff=yloff
ii+=1
#plt.figure();plt.imshow(alphapic[:,:,0])
#plt.figure();plt.imshow(tilepic[:,:,0])
#plt.figure();plt.imshow(fingerpic[:,:,0])
#plt.figure();plt.imshow(heatpics[:,:,50])
#testupload ####
archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
prog=IR.get_program_from_PID(program)
if prog[0]:
t1=prog[1]['trigger']['1'][0]
time_ns=[int(t1-1e9)]
for t3 in time:
time_ns.append(int(t1+t3*1e9))
images=np.append(alphapic,heatpics,axis=2)
chanDesc=[{'name':'heatflux','physicalQuantity':{'type':'W/m^2'},'active':1}]
parobj={"label":"parms",
"description":"heat flux for IR camera in AEF"+str(port),
"values":[{"chanDescs":chanDesc,'header':header,'comment':comment}],
"dimensions":[int(time_ns[0]),int(time_ns[-1])]}
url=archivepath+project+"/"+group+"/AEF"+str(port)+"_heatflux"
# url=IR.archivepath+project+"/"+group+"/"+stream
ver=IR.get_latest_version("Test/raw/"+project+"/"+group+"/"+stream+"_PARLOG",project=project,Test=True)
if ver==None:
reason="Versioning of Data"
IRup.archive_create_stream_version(url, reason, False, "n/a", "n/a" )
print(datetime.datetime.now(),"starting upload")
IRup.upload_Parlog(url,parobj,versionnumber=1)
print(datetime.datetime.now(),"parlog written")
IRup.upload_images(images,time=time_ns,group=group,stream=stream,project=project,version=1,swapping=False,Test=True)
print(datetime.datetime.now(),"data uploaded")
if upload_Mapping:
### upload reference stream ###once!
time_ns2=[]
timet1=int(IR.TimeToNs([2017,9,26],[8,0,0,0]))
for t in range(7):
time_ns2.append(int(timet1+10*t))
images2=np.append(profpic,Xpic,axis=2)
images2=np.append(images2,Ypic,axis=2)
images2=np.append(images2,FXpic,axis=2)
images2=np.append(images2,FYpic,axis=2)
images2=np.append(images2,fingerpic,axis=2) # images2=np.append(images2,profnpic,axis=2)
images2=np.append(images2,tilepic,axis=2)
header2={'profile':int(timet1),'MX':int(timet1+10),'MY':int(timet1+20),'FX':int(timet1+30),'FY':int(timet1+40),'finger_profile':int(timet1+50),'targetgroup':int(timet1+60)}
chanDesc=[{'name':'reference','physicalQuantity':{'type':'none'},'active':1}]
parobj={"label":"parms",
"description":"reference data for the heat flux",
"values":[{"chanDescs":chanDesc,'header':header2,'comment':comment}],
"dimensions":[int(time_ns2[0]),int(time_ns2[-1])]
}
# url=IR.archivepath+project+"/"+group+"/"+stream2
url=archivepath+project+"/"+group+"/"+stream2
ver=IR.get_latest_version("Test/raw/"+project+"/"+group+"/"+stream2+"_PARLOG",project=project,Test=True)
if ver==None:
reason="Versioning of Data"
IRup.archive_create_stream_version(url, reason, False, "n/a", "n/a" )
IRup.upload_Parlog(url,parobj,versionnumber=1)
IRup.upload_images(images2,time=time_ns2,group=group,stream=stream2,project=project,version=1,swapping=False,Test=True)
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 24 11:24:52 2017
Version: 2.9.5
Version: 3.0.0
@author: holn
"""
......@@ -920,7 +920,7 @@ def upload_fulldataset(url,parlog,data,versionnumber,bparlog=True,bdata=True,Ret
### from here on subfunctions
def upload_images(images,time,group,stream,newversion=False,reason="",project="W7X",swapping=True):
def upload_images(images,time,group,stream,newversion=False,reason="",project="W7X",version=0,swapping=True,Test=False):
"""
upload_images(camera,portnum,images,time,metadata={},newversion=False,reason="")
......@@ -930,12 +930,15 @@ def upload_images(images,time,group,stream,newversion=False,reason="",project="W
# names=[typ]
datastream_name=stream#"AEF"+str(portnum)+"_"+typ#+"_TEST001"
#url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
url=archivepath+project+"/"+group+"/"+datastream_name
if Test:
url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
else:
url=archivepath+project+"/"+group+"/"+datastream_name
if newversion:
archive_create_stream_version(url, reason, False, "n/a", "n/a" )
ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
version=ver
else:
elif version<=0 or version==None:
ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
if ver==None:
reason="Versioning of Data"
......@@ -945,7 +948,7 @@ def upload_images(images,time,group,stream,newversion=False,reason="",project="W
version=ver
try:
# archive.write_signals(images,time,group,project,names, datastream_name,units,metadata,2,True)
tmp=writeH5(datastream_name,images,time,swapping=swapping)
tmp=writeH5_images(datastream_name,images,time,swapping=swapping)
# uploadH5(group,datastream_name,tmp,True)
response=uploadH5_Versioning(group,datastream_name,tmp,version,True,project=project)
return response
......@@ -1026,10 +1029,33 @@ def writeH5(stream,data,dimof,idx=None,swapping=True):
data = data.swapaxes(0,1) # (width,height,time) -> (row,col,time)
with h5reader.File(tmpfile, 'w') as f:
g = f.create_group('data') # requires [row,col,time]
g.create_dataset('timestamps', data=list(dimof), dtype='uint64',compression="gzip")
g.create_dataset('timestamps', data=list(dimof), dtype='uint64',compression="gzip")
g.create_dataset(stream, data=data.tolist(), dtype=dtype,compression="gzip")
return tmpfile
def writeH5_images(stream,data,dimof,idx=None,swapping=True):
"""
path=Path, data=numpy.array, dimof=list of long
data.shape = (width,height,time) or (width,height) if len(dim)==1
"""
# stream = path.stream
dtype = str(data.dtype)
tmpfile = "archive_"+stream+'_'+str(dimof[0])
if idx: tmpfile += '_%d'%(idx,)
tmpfile += ".h5"
if data.ndim<3:
data = data.reshape(list(data.shape)+[1])
if swapping:
data = data.swapaxes(0,1) # (width,height,time) -> (row,col,time)
shapi=np.shape(data)
with h5reader.File(tmpfile, 'w') as f:
g = f.create_group('data') # requires [row,col,time]
g.create_dataset('timestamps', data=list(dimof), dtype='uint64',compression="gzip")
dset=g.create_dataset(stream,shape=(shapi[0],shapi[1],shapi[2]),dtype=dtype,chunks=(shapi[0],shapi[1],1))
data=np.ascontiguousarray(data)
dset.write_direct(data)
return tmpfile
#def writeH5_V2(stream,dataname,data,dimof,idx=None):
# """
# path=Path, data=numpy.array, dimof=list of long
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment