Skip to content
Snippets Groups Projects
upload_heatflux_folder.py 12.14 KiB
# -*- coding: utf-8 -*-
"""
Created on Fri Sep 21 09:32:29 2018
V3.0.2
@author: holn
"""

import numpy as np
import scipy.io as sio
#import matplotlib.pyplot as plt
import h5py as h5
import uploadingversionIRdata as IRup
import downloadversionIRdata as IR
import datetime
import os
import sys
#125,190 vertical, horizontal *3 mm



exception_ports=[]
path="F:\\holn\\Documents\\Doktorarbeit\\RESULTS\\OP1.2b\\Fulldivertor_res\\DBM\\H\\"
path="F:\\holn\\Documents\\Doktorarbeit\\Python-Programme\\THEODOR\\Divertor\\data\\"
path="F:\\tmp\\upload_it\\"
outpfad="F:\\holn\\Documents\\Doktorarbeit\\DATEN\\Mapping\\LinesforStacks\\"
archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
project="W7XAnalysis"
group="QRT_IRCAM"
  
comment="Test upload"    
header={'code_version':4,'alpha_corrected':1}
stream2="Mapping_reference"
upload_Mapping=False

class Tee(object):
    def __init__(self, *files):
        self.files = files
    def write(self, obj):
        for f in self.files:
            f.write(obj)
            f.flush() # If you want the output to be visible immediately
    def flush(self) :
        for f in self.files:
            f.flush()
            
original = sys.stdout
start=datetime.datetime.now()
fout=open("upload_log_heatflux_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
#foutf=open("upload_log_failes_"+str(start.date())+"_"+str(start.hour)+"_"+str(start.minute)+"_"+str(start.second)+".txt",'w')
sys.stdout = Tee(sys.stdout, fout)

filelist=os.listdir(path)

for file in filelist:
    if "heatflux_V4.mat" in file:
        program=file.split("_")[0]
        port=int(file.split("_")[1].split("AEF")[1])
        prog=IR.get_program_from_PID(program)
        if prog[0]:
            t1=prog[1]['trigger']['1'][0]
            time_ns=[int(t1-1e9)]
            try:
#                import archivedb as AKF                
                exist,tt,fra=IR.download_heatflux_by_times(port,t1,int(t1+100*1e6),version=1,request=False)            
            except:
                exist=False
            if exist:
                print(datetime.datetime.now(),"Data already uploaded for AEF"+str(port)+" , "+program)                
            elif port not in exception_ports:
                stream="AEF"+str(port)+"_heatflux"
                print(datetime.datetime.now(),"loading "+program+"_AEF"+str(port)+"_heatflux_V4.mat")
                File=sio.loadmat(path+program+"_AEF"+str(port)+"_heatflux_V4.mat")
                time=np.asarray(File['time'][0])
                tilelist=np.asarray(File['tiles'])
                heatpics=np.zeros((125+190+10,108*12+4,len(time)),dtype=np.float32)
                alphapic=np.zeros((125+190+10,108*12+4,1),dtype=np.float32)
                alphapic.fill(np.nan)
                heatpics.fill(np.nan)
                Xpic=alphapic.copy()
                Ypic=alphapic.copy()
                FXpic=alphapic.copy()
                FYpic=alphapic.copy()
                profpic=alphapic.copy()
                profnpic=alphapic.copy()
                tilepic=alphapic.copy()
                fingerpic=alphapic.copy()
                
                linedict={'h_l':30,'h_m':54,'h_h':24,'v':29}
                numstacks={            
                'h_l':{0:13,1:14,2:14,3:14,4:14,5:14,6:14,7:14,8:14,9:14,10:15,11:15,12:14,13:14,14:14,15:13,16:13,17:13,18:13,19:13,20:13,21:13,22:13,23:13,24:13,25:13,26:13,27:13,28:13,29:13},
                'h_m':{0:14,1:15,2:14,3:15,4:14,5:15,6:14,7:15,8:14,9:15,10:14,11:15,12:14,13:15,14:14,15:15,16:14,17:15,18:14,19:15,20:14,21:15,22:14,23:15,24:14,25:15,26:14,27:15,28:14,29:15,30:14,31:15,32:14,33:15,34:14,35:15,36:14,37:15,38:14,39:15,40:14,41:15,42:14,43:15,44:14,45:15,46:14,47:15,48:15,49:14,50:14,51:14,52:14,53:15},
                'h_h':{0:16,1:15,2:15,3:15,4:15,5:15,6:15,7:15,8:15,9:15,10:15,11:15,12:15,13:15,14:15,15:15,16:15,17:15,18:15,19:15,20:15,21:15,22:15,23:15},
                'v':{0:15,1:15,2:15,3:15,4:15,5:15,6:15,7:15,8:15,9:15,10:15,11:15,12:15,13:15,14:15,15:15,16:15,17:15,18:15,19:15,20:15,21:15,22:15,23:15,24:15,25:15,26:15,27:15,28:15}
                }
                
                def splitStacktoLines(locationall, rowcountarray):
                  locations = []
                  for i in range(len(rowcountarray) - 1):
                    locations.append(np.array(locationall[rowcountarray[i]:rowcountarray[i+1]]))
                  locations = np.array(locations)
                  return locations 
                ii=0
                V_index=0
                xoff=0
                maxyoff=0
                tn=0
                for tile in tilelist:#['h_l','h_m','h_h','v']:
                    tn+=1
                    tile=tile.replace(" ","")
                    linn=linedict[tile]
                    if tile!='v':
                        V_index+=linn
                        yoff=130
                    else:        
                        yoff=1
                        xoff=1
                    for i in range(linn):
                        if ii<30:
                            iko=ii
                        elif ii<84:
                            iko=ii-30            
                        elif ii<108:
                            iko=ii-84
                        else:
                            iko=ii-108
                        if ii>=78 and ii<108:
                            tn=3
                        fingerfile=str(port//10)
                        if port % 10 ==0:
                             fingerfile=fingerfile+"l"+tile+"_"
                        else:
                             fingerfile=fingerfile+"u"+tile+"_"
                        if port==20 and int(program.split(".")[0])<20171020:
                            linefile=fingerfile+'{0:02d}'.format(iko)+"2Dlines_biliniar_before_20171020.h5"
                        else:
                            linefile=fingerfile+'{0:02d}'.format(iko)+"2Dlines_biliniar.h5"
                        linf=h5.File(outpfad+linefile,'r')
                        linid=list(linf['LineNos'])
                        lines=linf['locationall2D']#
                        linlocid=linf['rowcountarray']
                        locations=splitStacktoLines(lines,linlocid)
                        linf.close()
                        xoff+=1
                        if tile=='h_m' and iko<48 and iko%2==1:
                            shift=True
                            xoff=xoff-numstacks[tile][i-1]-1
                            maxyoff+=1
                        else:
                            shift=False
                            maxyoff=0
                        for j in range(numstacks[tile][i]): 
                            j2=linid.index(j)
                            if shift:
                                yloff=maxyoff
                            else:
                                yloff=0
                            lprof=np.array(File['profiles_'+str(ii)+"_"+str(j2)][0])
                            lalpha=np.array(File['alpha_'+str(ii)+"_"+str(j2)][0])
                            lheat=np.array(File['heat_'+str(ii)+"_"+str(j2)])
                            lXY=np.array(File['plot_profiles_'+str(ii)+"_"+str(j2)])
                            loco=locations[j2]
                            profilelength=int(np.max(lprof)/3e-3)+1
                            for k in range(profilelength):                
                                pos=k*3e-3#3.143e-3                                        
                                pid=IR.IR_tools.find_nearest(lprof,pos)
                                if abs(pos-lprof[pid])<2e-3:
                                    alphapic[yoff+yloff,xoff]=lalpha[pid]
                                    heatpics[yoff+yloff,xoff,:]=lheat[:,pid]
                                    Xpic[yoff+yloff,xoff]=lXY[pid,0]
                                    Ypic[yoff+yloff,xoff]=lXY[pid,1]
                                    FXpic[yoff+yloff,xoff]=loco[pid,0]
                                    FYpic[yoff+yloff,xoff]=loco[pid,1]
                                    profpic[yoff+yloff,xoff]=lprof[pid]
                #                    profnpic[yoff+yloff,xoff]=j
                                    tilepic[yoff+yloff,xoff]=tn
                                    fingerpic[yoff+yloff,xoff]=ii*100+j
                                yloff+=1
                            xoff+=1
                            if yloff>maxyoff and not shift:
                                maxyoff=yloff
                        ii+=1
                del lalpha,lheat,lXY,loco,lprof,tn,File
                #plt.figure();plt.imshow(alphapic[:,:,0])
                #plt.figure();plt.imshow(tilepic[:,:,0])
                #plt.figure();plt.imshow(fingerpic[:,:,0])
                #plt.figure();plt.imshow(heatpics[:,:,50])
                #testupload ####
                print(datetime.datetime.now(),"loading and preparation done, finding program and starting upload")            
                for t3 in time:
                    time_ns.append(int(t1+t3*1e9))
                images=np.append(alphapic,heatpics,axis=2)
                chanDesc=[{'name':'heatflux','physicalQuantity':{'type':'W/m^2'},'active':1}]
                parobj={"label":"parms",
                            "description":"heat flux for IR camera in AEF"+str(port),
                            "values":[{"chanDescs":chanDesc,'header':header,'comment':comment}],
                            "dimensions":[int(time_ns[0]),int(time_ns[-1])]}                
                url=archivepath+project+"/"+group+"/AEF"+str(port)+"_heatflux"
            #    url=IR.archivepath+project+"/"+group+"/"+stream
                ver=IR.get_latest_version(group+"/"+stream+"_PARLOG",project=project,Test=True)
                if ver==None:
                    reason="Versioning of Data"
                    IRup.archive_create_stream_version(url, reason, False, "n/a", "n/a" )
                print(datetime.datetime.now(),"starting upload")
                IRup.upload_Parlog(url,parobj,versionnumber=1)  
                print(datetime.datetime.now(),"parlog written")
                IRup.upload_images(images,time=time_ns,group=group,stream=stream,project=project,version=1,swapping=False,Test=True)
                print(datetime.datetime.now(),"data uploaded")
                del images,alphapic,heatpics,time_ns
                if upload_Mapping:
                    ### upload reference stream ###once!
                    time_ns2=[]
                    timet1=int(IR.TimeToNs([2017,9,26],[8,0,0,0]))
                    for t in range(7):
                        time_ns2.append(int(timet1+10*t))
                    images2=np.append(profpic,Xpic,axis=2)
                    images2=np.append(images2,Ypic,axis=2)
                    images2=np.append(images2,FXpic,axis=2)
                    images2=np.append(images2,FYpic,axis=2)
                    images2=np.append(images2,fingerpic,axis=2)    #    images2=np.append(images2,profnpic,axis=2)    
                    images2=np.append(images2,tilepic,axis=2)
                    header2={'profile':int(timet1),'MX':int(timet1+10),'MY':int(timet1+20),'FX':int(timet1+30),'FY':int(timet1+40),'finger_profile':int(timet1+50),'targetgroup':int(timet1+60)}
                    chanDesc=[{'name':'reference','physicalQuantity':{'type':'none'},'active':1}]
                    parobj={"label":"parms",
                                "description":"reference data for the heat flux",
                                "values":[{"chanDescs":chanDesc,'header':header2,'comment':comment}],
                                "dimensions":[int(time_ns2[0]),int(time_ns2[-1])]
                                }
                #    url=IR.archivepath+project+"/"+group+"/"+stream2
                    url=archivepath+project+"/"+group+"/"+stream2
                    ver=IR.get_latest_version(group+"/"+stream2+"_PARLOG",project=project,Test=True)
                    if ver==None:
                        reason="Versioning of Data"
                        IRup.archive_create_stream_version(url, reason, False, "n/a", "n/a" )
                    IRup.upload_Parlog(url,parobj,versionnumber=1)  
                    IRup.upload_images(images2,time=time_ns2,group=group,stream=stream2,project=project,version=1,swapping=False,Test=True)
fout.close()
sys.stdout = original