Commit 13971823 authored by Holger Niemann's avatar Holger Niemann
Browse files

update to 2.7.0 option for upload and download implement to switch between...

update to 2.7.0 option for upload and download implement to switch between test and productive archive
parent d38e65d7
# -*- coding: utf-8 -*-
"""
Created on Tue Jun 5 09:26:06 2018
Version: 2.7.0
@author: holn
constants and config parameters
"""
#%% Paths
archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/W7X/"
calibpath="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Calibrations\\"
IRCamColdframes_fittingpath=calibpath+"IRCamColdframes_fitting\\"
IRCamRefImagespath = calibpath+'IRCamReferenceImages\\'
IRCAMBadPixels_path = calibpath + 'IRCAMBadPixels\\'
try:
# import sys
path=str(repr(__file__)).split("IR_config_constants")[0].split("'")[1]+"upload_config"
cFile=open(path.replace('\\\\',"\\"))#+"upload_config")
for line in cFile:
if line[0:3]!="###":
if line.split("=")[0]=='archivedb':
if eval(line.split("=")[1].split("\n")[0]):
archivepath="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/"
else:
archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
cFile.close()
except Exception as E:
archivepath="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"
print("config loading failed",E)
# raise Exception("Config file was not loaded properly",E)
#%% Dictionaries
portcamdict={
......
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 26 16:01:06 2017
Version: 2.5.3
Version: 2.7.0
@author: holn
"""
......
......@@ -3,7 +3,7 @@
Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018
Version: 2.6.1
Version: 2.7.0
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2) . #of updates to add functionalities . #number of updates for bug fixes )
@author: holn
"""
......@@ -66,13 +66,13 @@ def get_OP_by_time(time_ns):
else:
return None
def get_latest_version(stream):
def get_latest_version(stream,project="W7X"):
"""Find out latest version of given stream
:param stream url stream of interest
:return int of version number of None if non-versioned or non-existing stream
author: G. Schlisio, edit by holn
"""
request =urllib.request.Request(archivepath + stream + "/_versions.json", headers={"Accept": "application/json"})
request =urllib.request.Request(archivepath +project+"/" + stream + "/_versions.json", headers={"Accept": "application/json"})
try:
response = urllib.request.urlopen(request)
d = json.loads(response.read().decode('utf-8'))
......@@ -151,6 +151,7 @@ def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, ve
time in ns
Have to swap 11, 21 until correction in the database
"""
project="W7X"
OP=IR_tools.get_OP_by_time(time_ns=time)
if port == 21:
port = 11
......@@ -158,11 +159,11 @@ def download_LUT(port,time,exposure=0,emissivity=0,camera_filter=0,version=0, ve
port = 21
if port==50 and OP=="OP1.2a":# camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
stream="QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
larchivepath=archivepath+stream#"QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
larchivepath=archivepath+project+"/"+stream#"QRT_INFRATEC/"+"AEF"+str(port)+"_LUT_"
query="Filter_"+str(camera_filter)+"_Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
elif port in [10,11,20,21,30,31,40,41,51] or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
stream="QRT_IRCAM/"+"AEF"+str(port)+"_LUT_"
larchivepath=archivepath+stream
larchivepath=archivepath+project+"/"+stream
query="Texp_"+str(int(exposure))+"us_e_"+str(float(emissivity))
else:
print("camera unknown, stopping here")
......@@ -228,8 +229,8 @@ def download_NUC_by_program(port,program,exposure,version=0):
return False,0,0
def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_NUC_"
project="W7X"
larchivepath=archivepath+project+"/"+"QRT_IRCAM/AEF"+str(port)+"_NUC_"
# NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if version==0:
version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_NUC_DATASTREAM")
......@@ -398,7 +399,7 @@ def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametyp
t0date=t0date.isoformat()
if version==0:
version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_raw_DATASTREAM")
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," "))
t_offset=timest0[-1][0]-time_t[0]
if t_offset>10e6:
......@@ -409,7 +410,7 @@ def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametyp
return download_raw_images_by_times(port,ref_t[0]+t_offset,ref_t[1]+t_offset)
def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0):
larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_raw_"
larchivepath=archivepath+"W7X/QRT_IRCAM/AEF"+str(port)+"_raw_"
# NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if version==0:
version=get_latest_version("QRT_IRCAM/AEF"+str(port)+"_raw_DATASTREAM")
......@@ -470,10 +471,10 @@ def download_background_by_program(port,program,exposure,version=0):
def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0):
if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
stream="QRT_INFRATEC/"+"AEF"+str(port)+"_background_"
larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_background_"
larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_background_"
elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
stream="QRT_IRCAM/"+"AEF"+str(port)+"_background_"
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_background_"
larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_background_"
else:
print("camera unknown, stopping here")
raise Exception
......@@ -540,7 +541,9 @@ def download_raw_images_by_program(port,program,time_s=0,version=0,verbose=0):
# if prog[0]:
starttime=prog[0]['trigger']['1'][0]
stoptime=prog[0]['trigger']['6'][0]
if type(time_s)==list:
if type(time_s)==list or type(time_s)==np.ndarray:
if len(time_s)>2 or time_s[0]>1000:
raise Exception("invalid input for the timewindow!")
tstart=time_s[0]
tstop=time_s[1]
if tstop<tstart:
......@@ -562,11 +565,11 @@ def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=
if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
if version==0:
version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
if version==0:
version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
else:
if verbose>0:
print("camera unknown, stopping here")
......@@ -574,7 +577,7 @@ def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=
elif OP=="OP1.2b":
if version==0:
version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
if (stoptime-starttime)/intervalSize>1:
nrinterv=int(np.ceil((stoptime-starttime)/intervalSize))
......@@ -797,11 +800,11 @@ def download_last_raw_image_by_time(port,starttime,stoptime,version=0):
if Cam.split("_")[0]=="Infratec" and OP=="OP1.2a":#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
if version==0:
version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM")
larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
elif Cam.split("_")[0]=="IRCam"or OP=="OP1.2b":#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
if version==0:
version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM")
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_DATASTREAM/V"+str(version)+"/0/raw"
else:
raise Exception("camera unknown, stopping here")
try:
......@@ -832,11 +835,11 @@ def download_raw_parlog_by_times(port,starttime,stoptime,version=0):
if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
if version==0:
version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG")
larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)
larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)
elif port in [10,11,20,21,30,31,40,41,51]:#elif camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
if version==0:
version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG")
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)
larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG/V"+str(version)
else:
print("camera unknown, stopping here")
raise Exception
......@@ -870,11 +873,11 @@ def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0):
if OP=="OP1.2a":
if version==0:
version=get_latest_version("QRT_INFRATEC/"+"AEF"+str(port)+"_raw_PARLOG")
larchivepath=archivepath+"QRT_INFRATEC/AEF"+str(port)+"_raw_PARLOG/V"+str(version)
larchivepath=archivepath+"W7X/QRT_INFRATEC/AEF"+str(port)+"_raw_PARLOG/V"+str(version)
elif OP=="OP1.2b":
if version==0:
version=get_latest_version("QRT_IRCAM/"+"AEF"+str(port)+"_raw_PARLOG")
larchivepath=archivepath+"QRT_IRCAM/AEF"+str(port)+"_raw_PARLOG/V"+str(version)
larchivepath=archivepath+"W7X/QRT_IRCAM/AEF"+str(port)+"_raw_PARLOG/V"+str(version)
try:
res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
signal_list = json.loads(res.read().decode('utf-8'))
......@@ -906,11 +909,11 @@ def get_exposure_by_times(port,starttime,stoptime,version=0):
OP=IR_tools.get_OP_by_time(time_ns=stoptime)
if OP=="OP1.2a" and port==50:
# if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
# elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
else:
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
......@@ -932,10 +935,10 @@ def get_camera_temp_by_program(port,program,version=0):
def get_camera_temp_by_times(port,starttime,stoptime,version=0):
if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
else:
print("Port unknown")
raise Exception
......@@ -963,10 +966,10 @@ def get_sensor_temp_by_program(port,program,version=0):
def get_sensor_temp_by_times(port,starttime,stoptime,version=0):
if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
......@@ -988,10 +991,10 @@ def get_camera_framerate_by_program(port,program,version=0):
def get_camera_framerate_by_times(port,starttime,stoptime,version=0):
if port==50:#camera=="INFRATEC" or camera=="infratec" or camera=="Infratec":
stream="QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"W7X/QRT_INFRATEC/"+"AEF"+str(port)+"_meta_"
elif port in [10,11,20,21,30,31,40,41,51]:#camera=="IRCAM" or camera=="IRcam" or camera=="ircam":
stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
......@@ -1012,7 +1015,7 @@ def get_frametype_by_program(port,program,version=0):
def get_frametype_by_times(port,starttime,stoptime,version=0):
stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
larchivepath=archivepath+"W7X/QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/5/frametype/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
......
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 24 11:24:52 2017
Version: 2.5.3
Version: 2.7.0
@author: holn
"""
......@@ -9,11 +9,9 @@ Version: 2.5.3
import urllib
import json
#from getfoldernames import Searchdatalocation as gfn
import numpy as np
#from binfilesreadingv2 import BinVideoReader as bvreader
import os
import datetime as dt
#import datetime as dt
from IR_config_constants import archivepath,portcamdict
from IR_image_tools import get_OP_by_time
......@@ -147,8 +145,7 @@ def sendparlogjsonfileonly(url , jsondata , nameofdatastream ) :
from here on the uploadroutines from holn
---------------------------------------------------------------------------------------------------------------------------
"""
#import archive
project="W7X"
###definition of interactions with the codac archive to get times and other needed functions
import datetime
......@@ -289,9 +286,10 @@ def upload_meta_stream(camera,portnum,time,sensortemp,cameratemp,framerate,expos
old: sensortemp,cameratemp,framerate,exposuretime
new: channels with names, units and values
"""
project="W7X"
group="QRT_IRCAM"#+camera#+"_TEST"
datastream_name="AEF"+str(portnum)+"_meta"
url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
url=archivepath+project+"/"+group+"/"+datastream_name
if newversion:
archive_create_stream_version(url, reason, False, "n/a", "n/a" )
ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
......@@ -330,10 +328,11 @@ def upload_meta_stream_V2(camera,portnum,time,channel_names,channel_units,channe
old: sensortemp,cameratemp,framerate,exposuretime
new: channels with names, units and values
"""
project="W7X"
if len(time)==len(channel_values[0]) and len(channel_names)==len(channel_units):
group="QRT_IRCAM"#+camera#+"_TEST"
datastream_name="AEF"+str(portnum)+"_meta"
url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
url=archivepath+project+"/"+group+"/"+datastream_name
if newversion:
archive_create_stream_version(url, reason, False, "n/a", "n/a" )
ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
......@@ -380,6 +379,7 @@ def upload_raw_data(camera,portnum,time,images,metadata={},comment="",newversion
(bool) newversion: defines whether a new version is needed
(str) reason: reason for the new version
"""
project="W7X"
group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
datastream_name="AEF"+str(portnum)+"_raw"#_TEST_001"
chanDesc=[{'name':'raw','physicalQuantity':{'type':'none'},'active':1}]
......@@ -392,7 +392,7 @@ def upload_raw_data(camera,portnum,time,images,metadata={},comment="",newversion
"values":[{"chanDescs":chanDesc,'meta-data':metadata,'comment':comment}],
"dimensions":[time.tolist()[0],time.tolist()[-1]]
}
url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
url=archivepath+project+"/"+group+"/"+datastream_name
if newversion:
archive_create_stream_version(url, reason, False, "n/a", "n/a" )
ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
......@@ -407,7 +407,8 @@ def upload_raw_data(camera,portnum,time,images,metadata={},comment="",newversion
version=ver
print("writing parlog")
upload_Parlog(url,parobj,version)
upload_images(camera,portnum,images,time,"raw",newversion,reason)
#camera,portnum,,"raw"
upload_images(images,time,group,datastream_name,newversion,reason)
def regenerate_timevector(time,reftrig=1,tref=0,toff=0,give_program=False):
"""
......@@ -449,12 +450,13 @@ def upload_raw_data_from_H5(camera,portnum,time,h5file,metadata={},comment="",pa
(str) reason: reason for the new version
"""
#camera="IRCAM"
project="W7X"
group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
datastream_name="AEF"+str(portnum)+"_raw"#_TEST7"
#not tested#
chanDesc=[{'name':'raw','physicalQuantity':{'type':'none'},'active':1}]
#adjusted to metadataforcurrentvideo
url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
url=archivepath+project+"/"+group+"/"+datastream_name
if newversion:
archive_create_stream_version(url, reason, False, "n/a", "n/a" )
ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
......@@ -487,7 +489,7 @@ def upload_raw_data_from_H5(camera,portnum,time,h5file,metadata={},comment="",pa
success=True
print("uploading part "+str(n)+" of "+str(len(tmp)))
n+=1
bla= uploadH5_Versioning(group,datastream_name,ele,version,True)
bla= uploadH5_Versioning(group,datastream_name,ele,version,True,project=project)
if bla[0]:
print("upload of part "+str(n-1)+" of "+str(len(tmp))+" was successfull")
else:
......@@ -495,7 +497,7 @@ def upload_raw_data_from_H5(camera,portnum,time,h5file,metadata={},comment="",pa
print("upload of part "+str(n-1)+" of "+str(len(tmp))+" failed")
return [success],parlog_success
else:
return uploadH5_Versioning(group,datastream_name,tmp,version,True),parlog_success
return uploadH5_Versioning(group,datastream_name,tmp,version,True,project=project),parlog_success
def upload_temperaure_data(camera,portnum,images,time,metadata={},comment="",newversion=False,reason=""):
"""
......@@ -511,14 +513,21 @@ def upload_temperaure_data(camera,portnum,images,time,metadata={},comment="",new
"""
#not tested#
print("convert image data")
group="QRT_IRCAM"
datastream_name="AEF"+str(portnum)+"_temperature"
images=np.asarray(images,dtype=np.int16)
chanDesc=[{'name':'temperature','physicalQuantity':{'type':'K'},'active':1}]
parlog={'chanDescs':chanDesc,'meta-data':metadata,'comment':comment}
upload_images(camera,portnum,images,time,"temperature","K",parlog,newversion=False,reason="")
# upload_images(camera,portnum,images,time,"temperature","K",parlog,newversion=False,reason="")
url=archivepath+"W7X/"+group+"/"+datastream_name
upload_Parlog(url,parlog)
upload_images(images,time,group,datastream_name,newversion,reason)
def upload_Background_frame(camera,portnum,time,texp_filter,background,parlog=True,data=True,newversion=False,reason=""):
group="QRT_IRCAM"#+camera # test part will be removed in final version
# group="QRT_IRCAM_TEST" # test part will be removed in final version
project="W7X"
name=["background"]
datastream_name="AEF"+str(portnum)+"_background" # test part will be removed in final version
structure={
......@@ -526,7 +535,7 @@ def upload_Background_frame(camera,portnum,time,texp_filter,background,parlog=Tr
}
if camera=="INFRATEC":
structure['filter']=texp_filter[1]
url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
url=archivepath+project+"/"+group+"/"+datastream_name
if newversion:
archive_create_stream_version(url, reason, False, "n/a", "n/a" )
ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
......@@ -576,6 +585,7 @@ def upload_NUC_ref(camera,portnum,time,texp_filter,gain,offset,cold,bad,gain_e=[
#camera="IRCAM" #will be removed in final version
group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
name=["NUC"]
project="W7X"
datastream_name="AEF"+str(portnum)+"_NUC" # test part will be removed in final version
if np.shape(gain)==np.shape(bad):
badframe=bad
......@@ -605,7 +615,7 @@ def upload_NUC_ref(camera,portnum,time,texp_filter,gain,offset,cold,bad,gain_e=[
}
if camera=="INFRATEC":
structure['filter']=texp_filter[1]
url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
url=archivepath+project+"/"+group+"/"+datastream_name
if newversion:
archive_create_stream_version(url, reason, False, "n/a", "n/a" )
ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
......@@ -647,6 +657,7 @@ def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason=""
#camera="IRCAM" #will be removed in final version
group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
name=["LUT"]
project="W7X"
datastream_name="AEF"+str(portnum)+"_LUT"
#test of dataformat
if np.shape(LUTs)[0]!=len(time):
......@@ -674,7 +685,7 @@ def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason=""
exit()
if type(extra_par)==dict:
structure.update(extra_par)
url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
url=archivepath+project+"/"+group+"/"+datastream_name
if newversion:
archive_create_stream_version(url, reason, False, "n/a", "n/a" )
ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
......@@ -708,7 +719,8 @@ def upload_hot_cold_reference(port,path,time=1503907200000000000,newversion=Fals
"""
camera="IRCAM"
group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version
name=["raw"]
name=["raw"]
project="W7X"
### get the files in folder, sort it, check whether port in filename fit the expected setting
OP=get_OP_by_time(time_ns=time)
camera=portcamdict[OP]["AEF"+str(port)]
......@@ -751,7 +763,7 @@ def upload_hot_cold_reference(port,path,time=1503907200000000000,newversion=Fals
raise Warning("different file format detected! (differnt from the first one. it will be ignored")
###
datastream_name="AEF"+str(port)+"_raw"
url=archivepath+group+"/"+datastream_name
url=archivepath+project+"/"+group+"/"+datastream_name
if newversion:
archive_create_stream_version(url, reason, False, "n/a", "n/a" )
ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
......@@ -787,11 +799,12 @@ def upload_hot_cold_reference(port,path,time=1503907200000000000,newversion=Fals
def upload_scene_model(port,time_ns,path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Diagnostics\\QIR\\Calibrations\\scene_models",newversion=False,reason=""):
camera="IRCAM"
# camera="IRCAM"
project="W7X"
group="QRT_IRCAM_TEST"#+camera#+"_TEST" # test part will be removed in final version
name=["scene_model"]
datastream_name="AEF"+str(port)+"_scene_model"
url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
url=archivepath+project+"/"+group+"/"+datastream_name
Fullpath=os.path.join(path,"AEF"+str(port),"AEF"+str(port)+"_scene_model.h5")
File=h5reader.File(Fullpath,'r')
keylist=list(File.keys())
......@@ -824,7 +837,9 @@ def upload_scene_model(port,time_ns,path="\\\\sv-e4-fs-1\\E4-Mitarbeiter\\E4 Dia
"dimensions":[time_ns_list[0],time_ns_list[-1]]
}
upload_Parlog(url,parobj,version=version)
upload_images(camera,port,framelist,time_ns_list,"scene_model",version)
# upload_images(camera,port,framelist,time_ns_list,"scene_model",version)
upload_images(framelist,time_ns_list,group,datastream_name,newversion,reason)
#def upload_NUC_ref():
......@@ -898,7 +913,7 @@ def upload_fulldataset(url,parlog,data,versionnumber,bparlog=True,bdata=True,Ret
### from here on subfunctions
def upload_images(images,time,group,stream,newversion=False,reason=""):
def upload_images(images,time,group,stream,newversion=False,reason="",project="W7X",swapping=True):
"""
upload_images(camera,portnum,images,time,metadata={},newversion=False,reason="")
......@@ -908,7 +923,7 @@ def upload_images(images,time,group,stream,newversion=False,reason=""):
# names=[typ]
datastream_name=stream#"AEF"+str(portnum)+"_"+typ#+"_TEST001"
#url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
url="http://archive-webapi.ipp-hgw.mpg.de/Test/raw/"+project+"/"+group+"/"+datastream_name
url=archivepath+project+"/"+group+"/"+datastream_name
if newversion:
archive_create_stream_version(url, reason, False, "n/a", "n/a" )
ver=get_latest_version("Test/raw/"+project+"/"+group+"/"+datastream_name+"_PARLOG")
......@@ -923,18 +938,18 @@ def upload_images(images,time,group,stream,newversion=False,reason=""):
version=ver
try:
# archive.write_signals(images,time,group,project,names, datastream_name,units,metadata,2,True)
tmp=writeH5(datastream_name,images,time)
tmp=writeH5(datastream_name,images,time,swapping=swapping)
# uploadH5(group,datastream_name,tmp,True)
response=uploadH5_Versioning(group,datastream_name,tmp,version,True)
response=uploadH5_Versioning(group,datastream_name,tmp,version,True,project=project)
return response
except Exception as E:
print("Upload Failed")
print(E)
return [False]
def uploadH5(group,stream, h5file, delete=False, timeout=None):
def uploadH5(group,stream, h5file, delete=False, timeout=None,project="W7X"):
# path=Path, h5file=h5-file
base="http://archive-webapi/Test/raw/W7X/"
base=archivepath+project+"/"#"http://archive-webapi/Test/raw/W7X/"
try:
headers = {'Content-Type': 'application/x-hdf'}
link = base+group+'?dataPath=/data/'+stream+'&timePath=/data/timestamps'
......@@ -960,9 +975,9 @@ def uploadH5(group,stream, h5file, delete=False, timeout=None):
# _sup.debug(result,3)
# return result
def uploadH5_Versioning(group,stream, h5file,version, delete=False, timeout=None):
def uploadH5_Versioning(group,stream, h5file,version, delete=False, timeout=None,project="W7X"):
# path=Path, h5file=h5-file
base="http://archive-webapi/Test/raw/W7X/" #"http://sv-coda-wsvc-3/Test/raw/W7X/"#
base=archivepath+project+"/"#"http://archive-webapi/Test/raw/W7X/" #"http://sv-coda-wsvc-3/Test/raw/W7X/"#
#http://sv-coda-wsvc-3/Test/raw/W7X/QRT_IRCAM_TEST?dataPath=/data/AEF100_RAW_TEST2&timePath=/data/timestamps&versionNumber=1
try:
headers = {'Content-Type': 'application/x-hdf'}
......@@ -988,7 +1003,7 @@ def uploadH5_Versioning(group,stream, h5file,version, delete=False, timeout=None
print("successfully uploaded h5file")
return [True]
def writeH5(stream,data,dimof,idx=None):
def writeH5(stream,data,dimof,idx=None,swapping=True):
"""
path=Path, data=numpy.array, dimof=list of long
data.shape = (width,height,time) or (width,height) if len(dim)==1
......@@ -1000,7 +1015,8 @@ def writeH5(stream,data,dimof,idx=None):
tmpfile += ".h5"
if data.ndim<3:
data = data.reshape(list(data.shape)+[1])
data = data.swapaxes(0,1) # (width,height,time) -> (row,col,time)
if swapping:
data = data.swapaxes(0,1) # (width,height,time) -> (row,col,time)
with h5reader.File(tmpfile, 'w') as f:
g = f.create_group('data') # requires [row,col,time]
g.create_dataset('timestamps', data=list(dimof), dtype='uint64',compression="gzip")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment