Added the script to convert a plate of SDSS spectra into speccy format

convert_sdss_plate_to_speccy.csh

+#!/bin/tcsh -f
+
+# Task name:  convert_sdss_plate_to_speccy.csh
+# Author:     Tom Dwelly (dwelly AT mpe.mpg.de)
+# Function:   Convert all of the spectra from an SDSS plate into a set of "speccy" format files.
+# Inputs:     The "spPlate" and "spZbest" files that are assocated with a particular SDSS PLATE-MJD
+# Outputs:    A set of 1000 (one per fiber) 1D spectral files in "speccy" format. 
+#             Can be used with the "speccy" spectrum viewer: http://www.mpe.mpg.de/~tdwelly/speccy/speccy.html
+# Requirements: ftlist (from FTOOLS), awk, paste
+
+foreach HELPER ( ftlist awk paste ) 
+  if ( ! -X $HELPER ) then
+    echo "Error, cannot find $HELPER in your path" 
+    exit 1
+  endif
+end
+
+
+if ( $#argv != 3 ) then
+  echo "Error, incorrect number of input parameters"
+  echo "Usage: ${0:t} SPPLATE_FILENAME SPZBEST_FILENAME OUTDIR"
+  exit 1
+endif
+
+
+
+set SPPLATE = "${argv[1]}"
+set SPZBEST = "${argv[2]}"
+set OUTDIR  = "${argv[3]}"
+
+# check if the requisite files exist
+if ( ! -e "$SPPLATE" ) then
+  echo "Cannot find specified spPlate file: $SPPLATE"
+  exit 1
+endif
+
+if ( ! -e "$SPZBEST" ) then
+  echo "Cannot find specified spZbest file: $SPZBEST"
+  exit 1
+endif
+
+# make output directory for speccy format files:
+mkdir -p "$OUTDIR"
+if ( ! -d "$OUTDIR" ) then
+  echo "Cannot find/make output directory: $OUTDIR"
+  exit 1
+endif
+
+
+
+# set NFIBERS = 1000
+set NFIBERS =  `ftlist "${SPPLATE}[0]" K include="NAXIS2"  | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
+set PLATE   =  `ftlist "${SPPLATE}[0]" K include="PLATEID" | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
+set MJD     =  `ftlist "${SPPLATE}[0]" K include="MJD"     | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
+set CRVAL1  =  `ftlist "${SPPLATE}[0]" K include="CRVAL1"  | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
+set CRPIX1  =  `ftlist "${SPPLATE}[0]" K include="CRPIX1"  | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
+set CD1_1   =  `ftlist "${SPPLATE}[0]" K include="CD1_1"   | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
+set BUNIT   = (`ftlist "${SPPLATE}[0]" K include="BUNIT"   | awk '{print $3}'`)
+
+set TEMP0 = "${OUTDIR}/convert_to_speccy_temp0_$$"
+set TEMP1 = "${OUTDIR}/convert_to_speccy_temp1_$$"
+set TEMP2 = "${OUTDIR}/convert_to_speccy_temp2_$$"
+set TEMP3 = "${OUTDIR}/convert_to_speccy_temp3_$$"
+set TEMP4 = "${OUTDIR}/convert_to_speccy_temp4_$$"
+set TEMP5 = "${OUTDIR}/convert_to_speccy_temp5_$$"
+
+echo "Extracting speccy files from ${SPPLATE} and ${SPZBEST}"
+printf 'Output files will be at: %s/spec-%04d-%05d-{FIBERID}_speccy.txt\n'  "$OUTDIR" ${PLATE} ${MJD}
+echo -n "Working on FIBERID (total $NFIBERS): "
+ 
+set FIBERID = 1
+while ( $FIBERID <= $NFIBERS )
+  echo -n " $FIBERID"
+  set SPECCY_OUTFILE = `printf '%s/spec-%04d-%05d-%04d_speccy.txt'  "$OUTDIR" ${PLATE} ${MJD} ${FIBERID} `
+  rm $TEMP0 $TEMP1 $TEMP2 $TEMP3 $TEMP4 $TEMP5 >>& /dev/null
+
+  ftlist "${SPPLATE}[0][*,${FIBERID}:${FIBERID}]"    I rownum=no colheader=no mode=q | awk '//{for(i=1;i<=NF;i++){print $i}}' > $TEMP0
+  ftlist "${SPPLATE}[IVAR][*,${FIBERID}:${FIBERID}]" I rownum=no colheader=no mode=q | awk '//{for(i=1;i<=NF;i++){print $i}}' > $TEMP1
+  ftlist "${SPPLATE}[SKY][*,${FIBERID}:${FIBERID}]"  I rownum=no colheader=no mode=q | awk '//{for(i=1;i<=NF;i++){print $i}}' > $TEMP2
+  ftlist "${SPZBEST}[2][*,${FIBERID}:${FIBERID}]"    I rownum=no colheader=no mode=q | awk '//{for(i=1;i<=NF;i++){print $i}}' > $TEMP3
+      
+  # now dump the contents of the relevant tables in the SPZBEST and SPPLATE 
+  # files to ascii and format them them as speccy keywords (i.e. "# keyname = value")
+  ftlist "${SPZBEST}[1]" C mode=q | tail -n +4 | awk '//{print "KEYNAME",$2}' > $TEMP4
+  ftlist "${SPZBEST}[1][FIBERID==${FIBERID}]" T mode=q separator='|' rownum=no colheader=no > $TEMP5
+#  awk --field-separator='|' '$0!~/^KEYNAME/ {n++;if(n==1){nkey=NF;for(i=1;i<=nkey;i++){keyval[i]=$i}}} $0~/^KEYNAME/ {k++;split($0,a," ");keyname[k]=a[2]} END {for(i=1;i<=nkey;i++){printf("# %-20s = %s\n", keyname[i], keyval[i])}}' $TEMP4 $TEMP5 >  $SPECCY_OUTFILE
+#  awk --field-separator='|' '$0!~/^KEYNAME/ {n++;nkey=NF;for(i=1;i<=nkey;i++){n_i=sprintf("%d_%d",n,i);keyval[n_i]=$i}} $0~/^KEYNAME/ {k++;split($0,a," ");keyname[k]=a[2]} END {for(i=1;i<=nkey;i++){printf("# %-20s = ", keyname[i]);for(m=1;m<=n;m++){m_i=sprintf("%d_%d",m,i); printf("%s ",keyval[m_i])};printf("\n")}}' $TEMP4 $TEMP5 | awk --field-separator='' '//{for(m=1;m<=NF;m++){if($m!=" "){break;}};for(n=NF;n>0;n--){if($n!=" "){break;}};print substr($0,m,n-m+1)}' >  $SPECCY_OUTFILE
+
+  # deal with vector columns better
+  awk --field-separator='|' '\
+function trim_string(s) {\
+  for(__m=1;__m<=length(s);__m++){if(substr(s,__m,1)!=" "){break;}};\
+  for(__n=length(s);__n>0;__n--) {if(substr(s,__n,1)!=" "){break;}};\
+  return(substr(s,__m,__n-__m+1))}\
+$0!~/^KEYNAME/ {\
+  n++;nkey=NF;\
+  for(i=1;i<=nkey;i++){\
+    n_i=sprintf("%d_%d",n,i);\
+    keyval[n_i]=trim_string($i)}}\
+$0~/^KEYNAME/ {\
+  k++;split($0,a," ");keyname[k]=a[2]}\
+END {for(i=1;i<=nkey;i++){\
+  printf("# %-20s = ", keyname[i]);\
+  for(m=1;m<=n;m++){m_i=sprintf("%d_%d",m,i); printf("%s ",keyval[m_i])};printf("\n")}}' $TEMP4 $TEMP5 >  $SPECCY_OUTFILE
+
+  ftlist "${SPPLATE}[PLUGMAP]" C mode=q | tail -n +4 | awk '//{print "KEYNAME",$2}' > $TEMP4
+  ftlist "${SPPLATE}[PLUGMAP][FIBERID==${FIBERID}]" T mode=q separator='|' rownum=no colheader=no > $TEMP5
+  awk --field-separator='|' '$0!~/^KEYNAME/ {n++;if(n==1){nkey=NF;for(i=1;i<=nkey;i++){keyval[i]=$i}}} $0~/^KEYNAME/ {k++;split($0,a," ");keyname[k]=a[2]} END {for(i=1;i<=nkey;i++){printf("# %-20s = %s\n", keyname[i], keyval[i])}}' $TEMP4 $TEMP5 >>  $SPECCY_OUTFILE
+
+  paste $TEMP0 $TEMP1 $TEMP2 $TEMP3 $TEMP4 $TEMP5 | awk -v lcrval=$CRVAL1 -v crpix=${CRPIX1} -v lcdelt=${CD1_1} -v bunit="$BUNIT" 'BEGIN {printf("#Wavelength(AA,observed,vacuum),FluxDensity,Error,Sky,bestModel (all %s)\n", bunit)} //{i++;l=10.0**(lcrval + (i-crpix)*lcdelt);printf("%.2f,%.6g,%.6g,%.6g,%.6g\n", l, $1, ($2>0.0?$2**-0.5:0.0), $3, $4)} END {print _}' >> $SPECCY_OUTFILE
+  rm $TEMP0 $TEMP1 $TEMP2 $TEMP3 $TEMP4 $TEMP5 >>& /dev/null
+  @ FIBERID ++
+end
+echo " ... Done"
+
+
+
+exit
+