diff --git a/downloadversionIRdata.py b/downloadversionIRdata.py
index fb6e54cf260dc6a9954eafdb79089ee017bb34be..663e70ccfc623aa7f377044bc058b97894028edf 100644
--- a/downloadversionIRdata.py
+++ b/downloadversionIRdata.py
@@ -3,7 +3,7 @@
Created on Wed Oct 25 15:51:46 2017
updated on Tue Aug 21 10:20:00 2018
-Version: 2.5.3
+Version: 2.6.0
(Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2) . #of updates to add functionalities . #number of updates for bug fixes )
@author: holn
"""
@@ -11,10 +11,10 @@ Version: 2.5.3
import numpy as np
import IR_image_tools as IR_tools
from IR_config_constants import archivepath,portcamdict,camlimdict,valid_FOV_circle,valid_background_rectangle,TC_port,TC_channel,IRCamColdframes_fittingpath,exJet,exJet_trans
-try:
- import W7Xrest.read_restdb as AKF_1
-except:
- import read_restdb as AKF_1
+#try:
+# import W7Xrest.read_restdb as AKF_1
+#except:
+# import read_restdb as AKF_1
import datetime
import urllib
import json
@@ -214,7 +214,7 @@ def read_LUT_from_file(port, this_time, t_exp, emissivity, cfilter, emissivity_s
return False, []
def download_NUC_by_program(port,program,exposure,version=0):
- prog=AKF_1.get_program_from_PID(program)
+ prog=get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
@@ -251,7 +251,7 @@ def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
offset_i=NUCpar['structure']['offset_index']
n+=1
if nuctimes[0]!=0:
- NUC=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
+ NUC=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(nuctimes[0]-10)+"&upto="+str(nuctimes[1]+10))
if NUC[0]:
images=np.vsplit(NUC[2],np.shape(NUC[2])[0]/offset_i)
return True,images,['gain','offset','cold','badpixels','gain_error','offset_error']
@@ -265,7 +265,7 @@ def download_NUC_by_times(port,starttime,stoptime,exposure,version=0):
return False,0,0
def get_NUC_by_program(port,program,exposure,version=0,verbose=0):
- prog=AKF_1.get_program_from_PID(program)
+ prog=get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
@@ -435,11 +435,11 @@ def download_hot_cold_reference_by_times(port,exposure,starttime=150390720000000
print("hot image for requested exposure time not found")
images=[]
if HOTtime!=0:
- HOT=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
+ HOT=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(HOTtime-10)+"&upto="+str(HOTtime+10))
if HOT[0]:
images.append(HOT[2])
if COLDtime!=0:
- COLD=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
+ COLD=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(COLDtime-10)+"&upto="+str(COLDtime+10))
if COLD[0]:
images.append(COLD[2])
@@ -455,7 +455,7 @@ def download_background_by_program(port,program,exposure,version=0):
'''
returned exist,time,frame
'''
- prog=AKF_1.get_program_from_PID(program)
+ prog=get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
@@ -499,7 +499,7 @@ def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=
backtimes=[signal_list['dimensions'][2*n],signal_list['dimensions'][2*n+1]]
n+=1
if backtimes[0]!=0:
- backdat=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
+ backdat=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/_signal.json?from="+str(backtimes[0]-10)+"&upto="+str(backtimes[1]+10))
if backdat[0]:
return backdat#[True,backdat[2]]
else:
@@ -617,7 +617,7 @@ def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=
if fastDL:
def download_raw_images_by_program_via_png(port,program,time_s=0,version=0,threads=1, verbose=0):
- prog=AKF_1.get_program_from_PID(program)
+ prog=get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
@@ -848,7 +848,7 @@ def download_raw_parlog_by_times(port,starttime,stoptime,version=0):
def get_INFRATEC_filter_by_program(program,version=0):
- prog=AKF_1.get_program_from_PID(program)
+ prog=get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
@@ -884,7 +884,7 @@ def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0):
return False, 0,-1
def get_exposure_by_program(port,program,version=0):
- prog=AKF_1.get_program_from_PID(program)
+ prog=get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
@@ -909,7 +909,7 @@ def get_exposure_by_times(port,starttime,stoptime,version=0):
stream="QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
- return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/3/exposuretime"+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
+ return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/3/exposuretime"+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
def get_camera_temp_by_program(port,program,version=0):
# prog=AKF_1.get_program_from_PID(program)
@@ -936,14 +936,14 @@ def get_camera_temp_by_times(port,starttime,stoptime,version=0):
raise Exception
if version==0:
version=get_latest_version(stream+"DATASTREAM")
- dummy=AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
+ dummy=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
if dummy[0]:
return dummy
else:
- return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratempetarure/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
+ return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratempetarure/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
def get_sensor_temp_by_program(port,program,version=0):
- prog=AKF_1.get_program_from_PID(program)
+ prog=get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
@@ -964,10 +964,10 @@ def get_sensor_temp_by_times(port,starttime,stoptime,version=0):
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
- return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
+ return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
def get_camera_framerate_by_program(port,program,version=0):
- prog=AKF_1.get_program_from_PID(program)
+ prog=get_program_from_PID(program)
# try:
# t_program = AKF_2.get_program_from_to(program)
# prog =AKF_2.get_program_list(t_program[0], t_program[1])
@@ -989,7 +989,7 @@ def get_camera_framerate_by_times(port,starttime,stoptime,version=0):
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
- return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
+ return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
def get_frametype_by_program(port,program,version=0):
@@ -1010,7 +1010,7 @@ def get_frametype_by_times(port,starttime,stoptime,version=0):
larchivepath=archivepath+"QRT_IRCAM/"+"AEF"+str(port)+"_meta_"
if version==0:
version=get_latest_version(stream+"DATASTREAM")
- return AKF_1.read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/5/frametype/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
+ return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/5/frametype/_signal.json?from="+str(starttime)+"&upto="+str(stoptime))
def get_average_divertor_TC_for_camtemp(port,starttime,endtime,part="all",TC=0):
@@ -1072,7 +1072,7 @@ def get_average_divertor_TC_for_camtemp(port,starttime,endtime,part="all",TC=0):
i+=1
url=urlstart+str(offset+i)+ext+"0"+str(i)+"_TC"+str(i%8+1)
try:
- result=AKF_1.read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
+ result=read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
# res.append(result)
if result[0]:
n+=1
@@ -1131,7 +1131,7 @@ def get_TC_data_for_port(port,starttime,endtime,part=0,version=0):
for chi in range(len(thisTCchannel[0])):
url=urlstart+str(thisTCchannel[0][chi])+"/"+thisTCchannel[1][chi]
try:
- result=AKF_1.read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
+ result=read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
if result[0]:
n+=1
time=result[1]
@@ -1146,7 +1146,7 @@ def get_TC_data_for_port(port,starttime,endtime,part=0,version=0):
try:
url=urlstart+str(thisTCchannel[0][part-1])+"/"+thisTCchannel[1][part-1]
try:
- result=AKF_1.read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
+ result=read_restdb(url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
if result[0]:
n+=1
time=result[1]
@@ -1338,7 +1338,7 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver
# i=len(frames)
date=int(program.split(".")[0])
if date>20180712 and date<20180720:
- prog=AKF_1.get_program_from_PID(program)
+ prog=get_program_from_PID(program)
if prog[0]:
# t0time=prog[1]['trigger']['0'][0]
starttime=prog[1]['trigger']['1'][0]
@@ -1355,9 +1355,9 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_v
if threads==1 or not fastDL:
return get_temp_from_raw_by_program(portnr,program,time_s,emi,T_version,version,threads,give_ERROR)
else:
- background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version)
+ exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version)
FOV=get_FOV_mask(portnr)
- prog=AKF_1.get_program_from_PID(program)
+ prog=get_program_from_PID(program)
if prog[0]:
starttime=prog[1]['trigger']['1'][0]
stoptime=prog[1]['trigger']['6'][0]
@@ -1487,7 +1487,7 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_versi
else:
background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error=get_calib_data(portnr,program,emi,T_version,version)
FOV=get_FOV_mask(portnr)
- prog=AKF_1.get_program_from_PID(program)
+ prog=get_program_from_PID(program)
if prog[0]:
starttime=prog[1]['trigger']['1'][0]
stoptime=prog[1]['trigger']['6'][0]
@@ -1656,7 +1656,7 @@ def get_cooling_water_temperature_by_time(time_ns):
inlet_url="94/Ist%20Av06_ABK10_CT003"
url_base="http://archive-webapi.ipp-hgw.mpg.de/ArchiveDB/raw/W7X/CoDaStationDesc.85/DataModuleDesc.188_DATASTREAM/"
try:
- result_in=AKF_1.read_restdb(url_base+inlet_url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
+ result_in=read_restdb(url_base+inlet_url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
if result_in[0]:
inlet=np.mean(np.array(result_in[2]))
else:
@@ -1667,7 +1667,7 @@ def get_cooling_water_temperature_by_time(time_ns):
inlet=0
result_in=[False,0,0]
try:
- result_out=AKF_1.read_restdb(url_base+outlet_url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
+ result_out=read_restdb(url_base+outlet_url+"/_signal.json?from="+str(starttime)+"&upto="+str(endtime))
if result_out[0]:
outlet=np.mean(np.array(result_out[2]))
else:
@@ -1684,7 +1684,7 @@ def get_calib_data(port,program,emissivity=0.8,Temp_V=2,version=0,back_emissivit
get_calib_data
return exist,background,LUT,refT,gain,offset,badpixels,t_exp,cfilter,gain_error,offset_error
"""
- prog=AKF_1.get_program_from_PID(program)
+ prog=get_program_from_PID(program)
OP=IR_tools.get_OP_by_time(program_str=program)
exist=True
# try:
@@ -1973,6 +1973,111 @@ def estimate_offset(port,program,plot_it=False):
else:
return -1
+#%% general download functions
+"""
+the following functions are copied from W7xrest.read_restdb to remove this dependency
+@author: thomsen
+"""
+
+def read_restdb(request_url):
+ """
+ Reads JSON data from W7-X REST API
+ Returns:
+ valid: access ok
+ t: numpy-array of time
+ signal: numpy-array of requested signals
+
+ To get NaN, Infinity as their counter parts in Python (not 'Null')
+ add the string +'&allowNaN=true' in the request.
+ """
+ try:
+ res = urllib.request.urlopen(request_url)
+ signal_list = json.loads(res.read().decode('utf-8'))
+ res.close()
+ except urllib.error.URLError as e:
+ #if hasattr(e, 'reason'):
+ #print('Failed to reach server:',request_url)
+ #print('Reason: ', e.reason)
+ #elif hasattr(e, 'code'):
+ #print('The server couldn\'t fulfill the request.')
+ #print('Error code: ', e.code)
+ return False, [0], [-1]
+ else:
+# res = urllib.request.urlopen(request_url)
+# signal_raw=res.read()
+# signal_string=signal_raw.decode(encoding='UTF-8')
+# signal_list = json.loads(signal_string)
+
+ try:
+ signal0=np.array(signal_list['values'])
+ t=np.array(signal_list['dimensions'])
+ except ValueError as e:
+ print(signal_list['status'],e)
+ return False, [0], [-2]
+ else:
+ return True, np.squeeze(t), np.squeeze(np.double(signal0))
+
+def get_program_from_PID(instring):
+ """
+ Retrieve program information via json-request
+ (Python version of A.Alonso's Matlab routine getProgramInfo.m)
+ instring: specifies day and program ID (format yyyymmdd.pid)
+ Returns
+ valid: retrival ok
+ program_list: for program ID req_PID
+ (returns whole list, if req_PID was not found)
+
+ Example:
+ from W7Xrest import read_restdb as read_restdb
+ v,pl=read_restdb.get_program_from_PID('20160310.007')
+ if v:
+ print(pl['description'])
+ trigger6=pl['trigger']['6'][0]
+ """
+
+ try:
+ string_date,req_PID=instring.split('.')
+ except ValueError as e:
+ print('!get_program_from_PID: Format of input string should be similar to 20160310.007')
+ return False,0
+ try:
+ day=datetime.datetime.strptime(string_date, "%Y%m%d")
+ except ValueError as e:
+ print('!get_program_from_PID: Date format should be similar to 20160310')
+ return False,0
+
+ day_plus1=day+datetime.timedelta(days=1)
+ # create timestamps from begining to end of that day
+ w7xstamp0=np.int64(day.timestamp()*1e9)
+ w7xstamp1=np.int64(day_plus1.timestamp()*1e9)
+ # url to retrieve program information
+ program_url = 'http://archive-webapi.ipp-hgw.mpg.de/programs.json?from='
+ # write the url and load the data
+ jsonurl = program_url+np.str(w7xstamp0)+'&upto='+np.str(w7xstamp1)
+
+ try:
+ res = urllib.request.urlopen(jsonurl)
+ prog_list = json.loads(res.read().decode('utf-8'))
+ res.close()
+ except urllib.error.URLError as e:
+ print('!get_program_from_PID: Error opening URL')
+ return False,0
+ else:
+ pl=prog_list['programs']
+ id_list=[]
+ for pr in pl:
+ x=pr['id']
+ id_list.append(int((x.split('.'))[1]))
+ try:
+ id=id_list.index(int(req_PID))
+ except ValueError as e:
+ print('!get_program_from_PID: Could not find requested program ID',e)
+ return False,pl
+ else:
+ return True, pl[id]
+
+
+#%%
if __name__=='__main__':
print("local function calling")