-
Holger Niemann authoredHolger Niemann authored
upload_heatflux_folder.py 12.14 KiB
# -*- coding: utf-8 -*-
"""
Created on Fri Sep 21 09:32:29 2018
V3.0.2
@author: holn
"""
import numpy as np
import scipy.io as sio
#import matplotlib.pyplot as plt
import h5py as h5
import uploadingversionIRdata as IRup
import downloadversionIRdata as IR
import datetime
import os
import sys
#125,190 vertical, horizontal *3 mm
exception_ports=[]
path="F:\\holn\\Documents\\Doktorarbeit\\RESULTS\\OP1.2b\\Fulldivertor_res\\DBM\\H\\"
path="F:\\holn\\Documents\\Doktorarbeit\\Python-Programme\\THEODOR\\Divertor\\data\\"
path="F:\\tmp\\upload_it\\"
outpfad="F:\\holn\\Documents\\Doktorarbeit\\DATEN\\Mapping\\LinesforStacks\\"
archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
project="W7XAnalysis"
group="QRT_IRCAM"
comment="Test upload"
header={'code_version':4,'alpha_corrected':1}
stream2="Mapping_reference"
upload_Mapping=False
class Tee(object):
def __init__(self, *files):
self.files = files
def write(self, obj):
for f in self.files:
f.write(obj)
f.flush() # If you want the output to be visible immediately
def flush(self) :
for f in self.files:
f.flush()
original = sys.stdout
start=datetime.datetime.now()
fout=open("upload_log_heatflux_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
#foutf=open("upload_log_failes_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
sys.stdout = Tee(sys.stdout, fout)
filelist=os.listdir(path)
for file in filelist:
if "heatflux_V4.mat" in file:
program=file.split("_")[0]
port=int(file.split("_")[1].split("AEF")[1])
prog=IR.get_program_from_PID(program)
if prog[0]:
t1=prog[1]['trigger']['1'][0]
time_ns=[int(t1-1e9)]
try:
# import archivedb as AKF
exist,tt,fra=IR.download_heatflux_by_times(port,t1,int(t1+100*1e6),version=1,request=False)
except:
exist=False
if exist:
print(datetime.datetime.now(),"Data already uploaded for AEF"+str(port)+" , "+program)
elif port not in exception_ports:
stream="AEF"+str(port)+"_heatflux"
print(datetime.datetime.now(),"loading "+program+"_AEF"+str(port)+"_heatflux_V4.mat")
File=sio.loadmat(path+program+"_AEF"+str(port)+"_heatflux_V4.mat")
time=np.asarray(File['time'][0])
tilelist=np.asarray(File['tiles'])
heatpics=np.zeros((125+190+10,108*12+4,len(time)),dtype=np.float32)
alphapic=np.zeros((125+190+10,108*12+4,1),dtype=np.float32)
alphapic.fill(np.nan)
heatpics.fill(np.nan)
Xpic=alphapic.copy()
Ypic=alphapic.copy()
FXpic=alphapic.copy()
FYpic=alphapic.copy()
profpic=alphapic.copy()
profnpic=alphapic.copy()
tilepic=alphapic.copy()
fingerpic=alphapic.copy()
linedict={'h_l':30,'h_m':54,'h_h':24,'v':29}
numstacks={
'h_l':{0:13,1:14,2:14,3:14,4:14,5:14,6:14,7:14,8:14,9:14,10:15,11:15,12:14,13:14,14:14,15:13,16:13,17:13,18:13,19:13,20:13,21:13,22:13,23:13,24:13,25:13,26:13,27:13,28:13,29:13},
'h_m':{0:14,1:15,2:14,3:15,4:14,5:15,6:14,7:15,8:14,9:15,10:14,11:15,12:14,13:15,14:14,15:15,16:14,17:15,18:14,19:15,20:14,21:15,22:14,23:15,24:14,25:15,26:14,27:15,28:14,29:15,30:14,31:15,32:14,33:15,34:14,35:15,36:14,37:15,38:14,39:15,40:14,41:15,42:14,43:15,44:14,45:15,46:14,47:15,48:15,49:14,50:14,51:14,52:14,53:15},
'h_h':{0:16,1:15,2:15,3:15,4:15,5:15,6:15,7:15,8:15,9:15,10:15,11:15,12:15,13:15,14:15,15:15,16:15,17:15,18:15,19:15,20:15,21:15,22:15,23:15},
'v':{0:15,1:15,2:15,3:15,4:15,5:15,6:15,7:15,8:15,9:15,10:15,11:15,12:15,13:15,14:15,15:15,16:15,17:15,18:15,19:15,20:15,21:15,22:15,23:15,24:15,25:15,26:15,27:15,28:15}
}
def splitStacktoLines(locationall, rowcountarray):
locations = []
for i in range(len(rowcountarray) - 1):
locations.append(np.array(locationall[rowcountarray[i]:rowcountarray[i+1]]))
locations = np.array(locations)
return locations
ii=0
V_index=0
xoff=0
maxyoff=0
tn=0
for tile in tilelist:#['h_l','h_m','h_h','v']:
tn+=1
tile=tile.replace(" ","")
linn=linedict[tile]
if tile!='v':
V_index+=linn
yoff=130
else:
yoff=1
xoff=1
for i in range(linn):
if ii<30:
iko=ii
elif ii<84:
iko=ii-30
elif ii<108:
iko=ii-84
else:
iko=ii-108
if ii>=78 and ii<108:
tn=3
fingerfile=str(port//10)
if port % 10 ==0:
fingerfile=fingerfile+"l"+tile+"_"
else:
fingerfile=fingerfile+"u"+tile+"_"
if port==20 and int(program.split(".")[0])<20171020:
linefile=fingerfile+'{0:02d}'.format(iko)+"2Dlines_biliniar_before_20171020.h5"
else:
linefile=fingerfile+'{0:02d}'.format(iko)+"2Dlines_biliniar.h5"
linf=h5.File(outpfad+linefile,'r')
linid=list(linf['LineNos'])
lines=linf['locationall2D']#
linlocid=linf['rowcountarray']
locations=splitStacktoLines(lines,linlocid)
linf.close()
xoff+=1
if tile=='h_m' and iko<48 and iko%2==1:
shift=True
xoff=xoff-numstacks[tile][i-1]-1
maxyoff+=1
else:
shift=False
maxyoff=0
for j in range(numstacks[tile][i]):
j2=linid.index(j)
if shift:
yloff=maxyoff
else:
yloff=0
lprof=np.array(File['profiles_'+str(ii)+"_"+str(j2)][0])
lalpha=np.array(File['alpha_'+str(ii)+"_"+str(j2)][0])
lheat=np.array(File['heat_'+str(ii)+"_"+str(j2)])
lXY=np.array(File['plot_profiles_'+str(ii)+"_"+str(j2)])
loco=locations[j2]
profilelength=int(np.max(lprof)/3e-3)+1
for k in range(profilelength):
pos=k*3e-3#3.143e-3
pid=IR.IR_tools.find_nearest(lprof,pos)
if abs(pos-lprof[pid])<2e-3:
alphapic[yoff+yloff,xoff]=lalpha[pid]
heatpics[yoff+yloff,xoff,:]=lheat[:,pid]
Xpic[yoff+yloff,xoff]=lXY[pid,0]
Ypic[yoff+yloff,xoff]=lXY[pid,1]
FXpic[yoff+yloff,xoff]=loco[pid,0]
FYpic[yoff+yloff,xoff]=loco[pid,1]
profpic[yoff+yloff,xoff]=lprof[pid]
# profnpic[yoff+yloff,xoff]=j
tilepic[yoff+yloff,xoff]=tn
fingerpic[yoff+yloff,xoff]=ii*100+j
yloff+=1
xoff+=1
if yloff>maxyoff and not shift:
maxyoff=yloff
ii+=1
del lalpha,lheat,lXY,loco,lprof,tn,File
#plt.figure();plt.imshow(alphapic[:,:,0])
#plt.figure();plt.imshow(tilepic[:,:,0])
#plt.figure();plt.imshow(fingerpic[:,:,0])
#plt.figure();plt.imshow(heatpics[:,:,50])
#testupload ####
print(datetime.datetime.now(),"loading and preparation done, finding program and starting upload")
for t3 in time:
time_ns.append(int(t1+t3*1e9))
images=np.append(alphapic,heatpics,axis=2)
chanDesc=[{'name':'heatflux','physicalQuantity':{'type':'W/m^2'},'active':1}]
parobj={"label":"parms",
"description":"heat flux for IR camera in AEF"+str(port),
"values":[{"chanDescs":chanDesc,'header':header,'comment':comment}],
"dimensions":[int(time_ns[0]),int(time_ns[-1])]}
url=archivepath+project+"/"+group+"/AEF"+str(port)+"_heatflux"
# url=IR.archivepath+project+"/"+group+"/"+stream
ver=IR.get_latest_version(group+"/"+stream+"_PARLOG",project=project,Test=True)
if ver==None:
reason="Versioning of Data"
IRup.archive_create_stream_version(url, reason, False, "n/a", "n/a" )
print(datetime.datetime.now(),"starting upload")
IRup.upload_Parlog(url,parobj,versionnumber=1)
print(datetime.datetime.now(),"parlog written")
IRup.upload_images(images,time=time_ns,group=group,stream=stream,project=project,version=1,swapping=False,Test=True)
print(datetime.datetime.now(),"data uploaded")
del images,alphapic,heatpics,time_ns
if upload_Mapping:
### upload reference stream ###once!
time_ns2=[]
timet1=int(IR.TimeToNs([2017,9,26],[8,0,0,0]))
for t in range(7):
time_ns2.append(int(timet1+10*t))
images2=np.append(profpic,Xpic,axis=2)
images2=np.append(images2,Ypic,axis=2)
images2=np.append(images2,FXpic,axis=2)
images2=np.append(images2,FYpic,axis=2)
images2=np.append(images2,fingerpic,axis=2) # images2=np.append(images2,profnpic,axis=2)
images2=np.append(images2,tilepic,axis=2)
header2={'profile':int(timet1),'MX':int(timet1+10),'MY':int(timet1+20),'FX':int(timet1+30),'FY':int(timet1+40),'finger_profile':int(timet1+50),'targetgroup':int(timet1+60)}
chanDesc=[{'name':'reference','physicalQuantity':{'type':'none'},'active':1}]
parobj={"label":"parms",
"description":"reference data for the heat flux",
"values":[{"chanDescs":chanDesc,'header':header2,'comment':comment}],
"dimensions":[int(time_ns2[0]),int(time_ns2[-1])]
}
# url=IR.archivepath+project+"/"+group+"/"+stream2
url=archivepath+project+"/"+group+"/"+stream2
ver=IR.get_latest_version(group+"/"+stream2+"_PARLOG",project=project,Test=True)
if ver==None:
reason="Versioning of Data"
IRup.archive_create_stream_version(url, reason, False, "n/a", "n/a" )
IRup.upload_Parlog(url,parobj,versionnumber=1)
IRup.upload_images(images2,time=time_ns2,group=group,stream=stream2,project=project,version=1,swapping=False,Test=True)
fout.close()
sys.stdout = original