diff --git a/CHANGELOG b/CHANGELOG index 15e9d385c119c6dc33da666cb2ee0227b40522a4..e7fe1f47bbd5b7b611a8114e0c81714310c5b817 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -37,6 +37,12 @@ bugfixes in downloadversionIRdata: Versions: V3.0.0: download of heat flux data not also possible +V2.9.5: bug fixes for AEFXX, wrong dictionaries was used, use the correct defintion of the paths for the download. +V2.9.4: bug fixes for AEF50 download in OP1.2a (got trouble with the change to real Archive), fixes in the upload script to upload flir data and fix extra frames +V2.9.3: bugfixes for FLIR, ArchiveDB is now used as default as download +V2.9.2: bugfixes for AEF50 in the download +V2.9.1: bugfixes for AEF50 +V2.9.0: changed multiprocessing to multithreading, better performance, less archive calls V2.8.0: add FLIR camera into the downloadroutine, adapt the port interface, that it accept strings and numbers, numbers are automatically used for QRT V2.7.1: prove the temperature calculation, less ram usage V2.7.0: implement dependency on upload_config; switch between ArchiveDB and Test archive possible diff --git a/downloadversionIRdata.py b/downloadversionIRdata.py index d980e34776809829cd9b6865ff02716ace441b8f..360d7f60fd953202a57c452a73613f8cc6c23ad6 100644 --- a/downloadversionIRdata.py +++ b/downloadversionIRdata.py @@ -3,7 +3,7 @@ Created on Wed Oct 25 15:51:46 2017 updated on Tue Aug 21 10:20:00 2018 -Version: 2.9.4 +Version: 2.9.5 (Numbering: #of big changes(OP1.2a download V1, OP1.2b download V2, heatflux V3) . #of updates to add functionalities . #number of updates for bug fixes ) @author: holn """ @@ -227,10 +227,10 @@ def download_NUC_by_program(port,program,exposure,version=0): def download_NUC_by_times(port,starttime,stoptime,exposure,version=0): project="W7X" OP=get_OP_by_time(starttime) - larchivepath=archivepath+project+"/"+portcamdict[OP]["AEF"+str(port)]+"NUC_" + larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)]+"NUC_" # NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) if version==0: - version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM") + version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"NUC_DATASTREAM") try: res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) signal_list = json.loads(res.read().decode('utf-8')) @@ -396,8 +396,8 @@ def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametyp t0date=datetime.datetime.utcfromtimestamp((starttime-15e6)/1e9) t0date=t0date.isoformat() if version==0: - version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" timest0=AKF_2.get_time_intervals(larchivepath,t0date.replace("T"," "),t1date.replace("T"," ")) t_offset=timest0[-1][0]-time_t[0] if t_offset>10e6: @@ -409,10 +409,10 @@ def download_calibration_raw_files_by_time(port,t_exp,starttime,endtime,frametyp def download_hot_cold_reference_by_times(port,exposure,starttime=1503907200000000000,version=0): OP=IR_tools.get_OP_by_time(time_ns=starttime) - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_" + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_" # NUC_parlog=AKF_1.read_restdb_old(archivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) if version==0: - version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") + version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") try: res = urllib.request.urlopen(larchivepath+"PARLOG/V"+str(version)+"/_signal.json?from="+str(starttime)+"&upto="+str(int(starttime+1e9))) signal_list = json.loads(res.read().decode('utf-8')) @@ -468,8 +468,8 @@ def download_background_by_program(port,program,exposure,version=0): def download_background_by_times(port,starttime,stoptime,exposure,camera_filter=0,version=0): OP=IR_tools.get_OP_by_time(time_ns=starttime) - stream=portcamdict[OP]["AEF"+str(port)]+"background_" - larchivepath=archivepath+project+"/"+portcamdict[OP]["AEF"+str(port)] + stream=portpathdict[OP]["AEF"+str(port)]+"background_" + larchivepath=archivepath+project+"/"+portpathdict[OP]["AEF"+str(port)] if version==0: version=get_latest_version(stream+"DATASTREAM") try: @@ -507,7 +507,7 @@ def download_background_by_times(port,starttime,stoptime,exposure,camera_filter= def get_NUCed_background_by_times(port,t0,t1,t_exp,cfilter,gain,offset,version=0,plot_it=False): "OP1.2b function" exist,btime,backgroundframes=download_calibration_raw_files_by_time(port,t_exp,t0,t1,frametype=1,version=version) - camera=portcamdict["OP1.2b"]["AEF"+str(port)] + camera=portpathdict["OP1.2b"]["AEF"+str(port)] background=np.zeros(np.shape(backgroundframes[0]),dtype=np.uint64) for frame in backgroundframes: background+=frame @@ -554,8 +554,8 @@ def download_raw_images_by_program(port,program,time_s=0,version=0,verbose=0): def download_raw_images_by_times(port,starttime,stoptime,version=0,intervalSize=1E9,verbose=0): OP=IR_tools.get_OP_by_time(time_ns=starttime) if version==0: - version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" if (stoptime-starttime)/intervalSize>1: nrinterv=int(np.ceil((stoptime-starttime)/intervalSize)) if verbose>0: @@ -615,8 +615,8 @@ if fastDL: stoptime=prog[1]['trigger']['6'][0] OP=get_OP_by_time(starttime) if version==0: - version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" return download_raw_images_by_time_via_png(larchivepath,starttime,stoptime,time_s=time_s,threads=threads,version=version,verbose=verbose-1) def download_raw_images_by_time_via_png(larchivepath,starttime,stoptime,time_s=0,threads=1,version=0,verbose=0): @@ -827,10 +827,14 @@ def download_raw_FLIR_images_via_png(t1,t6,time_s=0,version=0,threads=1,verbose= if mode==1: ### okay the data is stored probably as different sets (so recorded with FLIR software) timess,versions=get_FLIR_Pset_times(stdate,enddate,version) + if verbose>0: + print("found ",np.shape(timess[0]),np.shape(timess[1]),np.shape(timess[2]),np.shape(timess[3])," frames in the different sets") ### we not know the times for all presets for the program, now we have to download them, if times is not empty exist,time,frames,tsets=download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=verbose-1) elif mode==2:##the data is stored only in one datastream, at least this datastream exist raise Exception("not implemented") + #download data and downlaod LUT + # time=[] # images=[] # lnt=len(times) @@ -849,7 +853,7 @@ def download_raw_FLIR_images_via_png(t1,t6,time_s=0,version=0,threads=1,verbose= return exist,time,frames,mode,tsets,versions -def get_FLIR_Pset_times(stdate,enddate,version=0): +def get_FLIR_Pset_times(stdate,enddate,version=0,verbose=0): larchivepath=archivepath+"W7X/QSR07_FLIR/AEK51_raw" if version==0: versions=[] @@ -871,7 +875,7 @@ def get_FLIR_Pset_times(stdate,enddate,version=0): times_0=AKF_2.get_time_intervals(larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw",stdate.replace("T"," "),enddate.replace("T"," "))# found=True except Exception as E: - print(E) + print(E,larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw") found=False times_0=[] version0=version0-1 @@ -884,7 +888,8 @@ def get_FLIR_Pset_times(stdate,enddate,version=0): counter+=1 try: times_0=AKF_2.get_time_intervals(larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw",stdate.replace("T"," "),enddate.replace("T"," "))# - except: + except Exception as E: + print(E,larchivepath+"_PS"+str(counter)+"_DATASTREAM/V"+str(version0)+"/0/raw") times_0=[] timess.append(times_0) return timess,versions @@ -930,13 +935,14 @@ def download_FLIR_Psets(larchivepath,timess,threads,versions,verbose=0): larchivepaths=[] versionss=[] for times in timess: + times=np.array(times) setn+=1 if times!=[]: counter+=1 threads+=1 Psets.append(counter) larchivepaths.append(larchivepath+"_PS"+str(setn)+"_DATASTREAM/V"+str(versions[setn])+"/0/raw") - tim.append(times) + tim.append(times[:,0]) versionss.append(versions[setn]) jobs = [] resultdict = [] @@ -996,7 +1002,7 @@ def download_last_raw_image_by_time(larchivepath,starttime,stoptime,version=0): pixelarray = np.array(img,dtype=np.uint16)#.swapaxes(0,1) return True, pixelarray except urllib.error.URLError as e: - print(e) + print(e,larchivepath+"/_signal.png?from="+str(starttime-10)+"&upto="+str(stoptime)) return False, [-1] def download_raw_parlog_by_program(port,program,version=0): @@ -1015,8 +1021,8 @@ def download_raw_parlog_by_program(port,program,version=0): def download_raw_parlog_by_times(port,starttime,stoptime,version=0): OP=IR_tools.get_OP_by_time(time_ns=stoptime) if version==0: - version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_PARLOG") - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version) + version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG") + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version) try: res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) signal_list = json.loads(res.read().decode('utf-8')) @@ -1045,8 +1051,8 @@ def get_INFRATEC_filter_by_program(program,version=0): def get_INFRATEC_filter_by_times(starttime,stoptime,port=50,version=0): OP=IR_tools.get_OP_by_time(time_ns=starttime) if version==0: - version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_PARLOG") - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version) + version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG") + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_PARLOG/V"+str(version) try: res = urllib.request.urlopen(larchivepath+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) signal_list = json.loads(res.read().decode('utf-8')) @@ -1076,8 +1082,8 @@ def get_exposure_by_program(port,program,version=0): def get_exposure_by_times(port,starttime,stoptime,version=0): OP=IR_tools.get_OP_by_time(time_ns=stoptime) - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"meta_" - stream=portcamdict[OP]["AEF"+str(port)]+"meta_" + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_" + stream=portpathdict[OP]["AEF"+str(port)]+"meta_" if version==0: version=get_latest_version(stream+"DATASTREAM") return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/3/exposuretime"+"/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) @@ -1097,8 +1103,8 @@ def get_camera_temp_by_program(port,program,version=0): def get_camera_temp_by_times(port,starttime,stoptime,version=0): OP=IR_tools.get_OP_by_time(time_ns=stoptime) - stream=portcamdict[OP]["AEF"+str(port)]+"meta_" - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"meta_" + stream=portpathdict[OP]["AEF"+str(port)]+"meta_" + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_" if version==0: version=get_latest_version(stream+"DATASTREAM") dummy=read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/1/cameratemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) @@ -1122,8 +1128,8 @@ def get_sensor_temp_by_program(port,program,version=0): def get_sensor_temp_by_times(port,starttime,stoptime,version=0): OP=IR_tools.get_OP_by_time(time_ns=stoptime) - stream=portcamdict[OP]["AEF"+str(port)]+"meta_" - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"meta_" + stream=portpathdict[OP]["AEF"+str(port)]+"meta_" + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_" if version==0: version=get_latest_version(stream+"DATASTREAM") return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/0/sensortemperature/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) @@ -1144,8 +1150,8 @@ def get_camera_framerate_by_program(port,program,version=0): def get_camera_framerate_by_times(port,starttime,stoptime,version=0): OP=IR_tools.get_OP_by_time(time_ns=stoptime) - stream=portcamdict[OP]["AEF"+str(port)]+"meta_" - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"meta_" + stream=portpathdict[OP]["AEF"+str(port)]+"meta_" + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_" if version==0: version=get_latest_version(stream+"DATASTREAM") return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/2/framerate/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) @@ -1166,8 +1172,8 @@ def get_frametype_by_program(port,program,version=0): def get_frametype_by_times(port,starttime,stoptime,version=0): OP=IR_tools.get_OP_by_time(time_ns=starttime) - stream=portcamdict[OP]["AEF"+str(port)]+"meta_" - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"meta_" + stream=portpathdict[OP]["AEF"+str(port)]+"meta_" + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_" if version==0: version=get_latest_version(stream+"DATASTREAM") return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/5/frametype/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) @@ -1187,8 +1193,8 @@ def get_framenumber_by_program(port,program,version=0): def get_framenumber_by_times(port,starttime,stoptime,version=0): OP=IR_tools.get_OP_by_time(time_ns=starttime) - stream=portcamdict[OP]["AEF"+str(port)]+"meta_" - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"meta_" + stream=portpathdict[OP]["AEF"+str(port)]+"meta_" + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"meta_" if version==0: version=get_latest_version(stream+"DATASTREAM") return read_restdb(larchivepath+"DATASTREAM/V"+str(version)+"/4/framenumber/_signal.json?from="+str(starttime)+"&upto="+str(stoptime)) @@ -1572,8 +1578,8 @@ def get_temp_from_raw_by_program(portnr,program,time_s=0,emi=0.8,T_version=2,ver return False,[0],[0],False else: if version==0: - version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" if verbose>0: print(datetime.datetime.now(),"raw download start") if fastDL: @@ -1651,8 +1657,8 @@ def get_temp_from_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_v success=True OP=get_OP_by_time(starttime) if version==0: - version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" if type(time_s)==list: tstart=time_s[0] tstop=time_s[1] @@ -1772,8 +1778,8 @@ def get_nuced_raw_by_program_fullthreads(portnr,program,time_s=0,emi=0.8,T_versi success=True OP=get_OP_by_time(t1) if version==0: - version=get_latest_version(portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") - larchivepath=archivepath+"W7X/"+portcamdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" + version=get_latest_version(portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM") + larchivepath=archivepath+"W7X/"+portpathdict[OP]["AEF"+str(port)]+"raw_DATASTREAM/V"+str(version)+"/0/raw" if type(time_s)==list: tstart=time_s[0] tstop=time_s[1] @@ -2355,9 +2361,9 @@ end of the copied functions if __name__=='__main__': print("local function calling") - port="AEF51" - prog="20180925.065" - status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,4],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False,verbose=5) + port=50#"AEF51" + prog="20180925.017" + status,time,images,valid=get_temp_from_raw_by_program(port,prog,time_s=[0,0.15],emi=0.82,T_version=2,version=0,threads=4,give_ERROR=False,use_firstframe_as_background=False,verbose=5) if status: plt.figure() plt.imshow(images[-1],vmin=330,vmax=1000,cmap=exJet) diff --git a/uploadingversionIRdata.py b/uploadingversionIRdata.py index 585300822315701e9c8f212077541d08712e66ad..317ffe437ea979d6c193b1cb6fbee711da5955f7 100644 --- a/uploadingversionIRdata.py +++ b/uploadingversionIRdata.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ Created on Thu Aug 24 11:24:52 2017 -Version: 2.9.4 +Version: 2.9.5 @author: holn """ @@ -643,7 +643,7 @@ def upload_NUC_ref(camera,portnum,time,texp_filter,gain,offset,cold,bad,gain_e=[ upload_fulldataset(url,parobj,dataobj,version,parlog,data) -def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason="",extra_par=None): +def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,group="QRT_IRCAM",newversion=False,reason="",extra_par=None): """ upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason="") (numpy.array) LUTs: t x n x 3 array DL,T,T_er, in general t=1, n= number of look-up-tables @@ -655,10 +655,15 @@ def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason="" (str) reason: reason for the new version """ #camera="IRCAM" #will be removed in final version - group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version +# group="QRT_IRCAM"#+camera#+"_TEST" # test part will be removed in final version name=["LUT"] project="W7X" - datastream_name="AEF"+str(portnum)+"_LUT" + if type(portnum)==int: + datastream_name="AEF"+str(portnum)+"_LUT" + descri="LUT for "+camera+" in AEF"+str(portnum) + elif type(portnum)==str: + datastream_name=portnum+"_LUT" + descri="LUT for "+camera+" in "+portnum #test of dataformat if np.shape(LUTs)[0]!=len(time): print("missmatch in between time vector and LUTs timestamps") @@ -678,6 +683,8 @@ def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason="" for n in range(np.shape(LUTref[0])[0]): if camera=="INFRATEC": structure["Filter_"+str(LUTref[0][n][0])+"_Texp_"+str(LUTref[0][n][1])+"us_e_"+str(LUTref[0][n][2])]=2*n+1 + elif camera=="FLIR": + structure["Texp_"+str(LUTref[0][n][0])+"us_e_"+str(LUTref[0][n][1])]=2*n+1 else: structure["Texp_"+str(LUTref[0][n][0])+"us_e_"+str(LUTref[0][n][1])]=2*n+1 else: @@ -701,7 +708,7 @@ def upload_LUT_tables(LUTs,LUTref,camera,portnum,time,newversion=False,reason="" #using here selfmade uploadroutine to avoid that LUTs table appear as images chanDesc={"[0]":{'name':name[0],'physicalQuantity':{'type':'none'},'active':1}} parobj={"label":"parms", - "description":"LUT for "+camera+" in AEF"+str(portnum), + "description":descri, "values":[{"chanDescs":chanDesc,"structure":structure}], "dimensions":[time[0],time[0]+10] }