readfrfs (version 3.2)
You can also look at:
#! /usr/bin/ksh
###########################################################################
#
# SYNTAX: readfrfs
#
# BRIEFLY: Unpack the raw telemetry into FITS event and housekeeping files.
#
# DESCRIPTION: This routine runs frfread to unpack the raw telemetry into
# DESCRIPTION: a large number of science files. A new science file is
# DESCRIPTION: created every time there is a mode change.
# DESCRIPTION: In addition frfread produces five housekeeping files:
# DESCRIPTION: one for each instrument and one general file.
# DESCRIPTION: <P>
# DESCRIPTION: After running frfread, this routine does the following:
# DESCRIPTION: <P>
# DESCRIPTION: All raw event files with no events are deleted.
# DESCRIPTION: Files with empty GTI extensions are also deleted, but
# DESCRIPTION: an error is given when this happens.
# DESCRIPTION: <P>
# DESCRIPTION: The TSTART and TSTOP keywords in the HK files are checked
# DESCRIPTION: for sensible values. frfread errors have sometimes caused
# DESCRIPTION: TSTART=0.0.
# DESCRIPTION: <P>
# DESCRIPTION: All of the good time intervals for a given instrument
# DESCRIPTION: are collected into a FITS extension which is appended to
# DESCRIPTION: the appropriate HK file. All GTIs which overlap the interval
# DESCRIPTION: between
# DESCRIPTION: TSTART and TSTOP of a particular HK file are included
# DESCRIPTION: in that file's extension.
# DESCRIPTION: <P>
#
# VERSION: 3.2
#
# HISTORY: 0.0 -> 1.0 1/3/97
# HISTORY: Corrected bug which fed the wrong exit code from frfread to
# HISTORY: ftool_test.
# HISTORY:
# HISTORY: 1.0 -> 1.1 2/11/97
# HISTORY: Changed frfread errors to level 2, now that they are less common.
# HISTORY: Moved check for zero TSTARTS and TSTOPS in HK files to appear
# HISTORY: immediately after frfread run.
# HISTORY: Added code to append ALLGTI extensions to the instrument HK files.
# HISTORY: Updated the description above.
# HISTORY: 2/15/97 Removed ALLGTI generating code until it has GOF approval.
# HISTORY: 2/17/97 Added code to save FRAME mode data for possible use
# HISTORY: in RDD correction.
# HISTORY: 2/18/97 Now reads ra and dec directly from job.par.
# HISTORY:
# HISTORY: 1.1 -> 1.2 2/26/97
# HISTORY: Uncommented the ALLGTI generating code.
# HISTORY: Changed name of frame mode files to avoid problems later.
# HISTORY:
# HISTORY: 1.2 -> 1.3 5/15/97
# HISTORY: Now removes zero and negative GTIs from ALLGTI extensions
# HISTORY: which are added to the HK files. This is cleaner, and
# HISTORY: keeps compute_exposure from complaining.
# HISTORY:
# HISTORY: 1.3 -> 1.4 6/11/97
# HISTORY: Added a check for raw files with empty GTI extensions.
# HISTORY: If one is found, the file is deleted and an exception is given.
# HISTORY: This indicates a likely bug in frfread, but this is the most graceful
# HISTORY: way to recover.
# HISTORY:
# HISTORY: 1.4 -> 1.5 6/19/97
# HISTORY: Removed some leftover code at the end which repeated the check
# HISTORY: for NEVENTS=0 in the event files.
# HISTORY:
# HISTORY: 1.5 -> 1.6 8/11/97
# HISTORY: Now uses Dotani's rddsoft software to extract FRAME mode data
# HISTORY: from the telemetry. Apparently frfread wasn't doing this correctly.
# HISTORY:
# HISTORY: 1.6 -> 1.7 9/16/97
# HISTORY: Filtered out some of the benign error messages from the FRAME
# HISTORY: mode extraction and processing software.
# HISTORY:
# HISTORY: 1.7 -> 1.8 9/25/97
# HISTORY: Now when checking raw files for NEVENTS=0 and non-empty GTI
# HISTORY: extensions, uses ffilecat to read the keywords instead of
# HISTORY: using read_fits_keyword. This should be much faster for large
# HISTORY: sequences.
# HISTORY:
# HISTORY: 1.8 -> 1.9 10/12/97
# HISTORY: Now checks if there are any event files before merging and
# HISTORY: appending GTIs.
# HISTORY:
# HISTORY: 1.9 -> 2.0 12/29/97
# HISTORY: Now includes all GTIs which have any overlap with TSTART and TSTOP
# HISTORY: of a given HK file when creating the ALLGTI extension.
# HISTORY: Before, only GTIS which were wholly contained were included.
# HISTORY: In at least one case this caused GTIs to be thrown away
# HISTORY: because of roundoff error.
# HISTORY:
# HISTORY: 2.0 -> 2.1 1/6/98
# HISTORY: Took out the frame mode extraction code and moved it to the
# HISTORY: framedata subroutine. This was done so that information could
# HISTORY: be extracted from the filter file and placed in the frame file
# HISTORY: headers.
# HISTORY:
# HISTORY: 2.1 -> 2.2 3/20/98
# HISTORY: Supplied missing continuation characters in fselect parameter
# HISTORY: setups.
# HISTORY: 2.2 -> 2.3 4/24/98
# HISTORY: Now code handles both gzipped and compressed telemetry files
# HISTORY:
# HISTORY: 2.3 -> 3.0 1998-08-06
# HISTORY: Modified to work with the new frfread 4.0
# HISTORY: Accomodated a change in the behavior in ffilecat which now
# HISTORY: keeps the extension specifier in the listed filenames.
# HISTORY:
# HISTORY: 3.0 -> 3.1 1999-05-05
# HISTORY: Modified the syntax in the calculation of the percentage
# HISTORY: of corrupted superframes to compensate for some subtlties
# HISTORY: in the operator precedence in $STOOL/equals.
# HISTORY:
# HISTORY: 3.1 -> 3.2 1999-11-09
# HISTORY: Now read criterion for corrupted superframe warning from
# HISTORY: process parfile instead of hard coding it.
# HISTORY: Also started filtering out warnings for out of order GIS events.
#
###########################################################################
#DEBUG=1
$UTIL/milestone "Running frfread on telemetry files"
####################################
# seqpi (according to the seq_type)
####################################
seq_type=$($UTIL/read_parfile $JOBPAR seq_type)
case "$seq_type" in
1 ) seqpi="$($UTIL/read_parfile $JOBPAR jppiname)" ;;
2 ) seqpi="$($UTIL/read_parfile $JOBPAR uspiname)" ;;
3 ) seqpi="$($UTIL/read_parfile $JOBPAR jppiname)" ;;
4 ) seqpi="$($UTIL/read_parfile $JOBPAR uspiname)" ;;
6 ) seqpi="$($UTIL/read_parfile $JOBPAR uspiname)" ;;
7 ) seqpi="$($UTIL/read_parfile $JOBPAR jppiname)" ;;
8 ) seqpi="$($UTIL/read_parfile $JOBPAR uspiname)" ;;
9 ) seqpi="Manager" ;;
11) seqpi="$($UTIL/read_parfile $JOBPAR uspiname)" ;;
* ) seqpi="Unknown PI"
echo $UTIL/exception $0 1 "Unknown sequence type $seq_type" ;;
esac
#############
# RA and Dec
#############
ra=$( $UTIL/read_parfile $JOBPAR ra )
dec=$($UTIL/read_parfile $JOBPAR dec)
#############################################
# criterion for warning about lost superframes
###############################################
lost_sf_percent=$($UTIL/read_parfile $PARFILE lost_sf_percent )
#####################################
# run frfread on all telemetry files
######################################
frfread=$($UTIL/read_parfile $PARFILE frfread)
SYSPFILES_save="$SYSPFILES"
PFILES_save="$PFILES"
rm -f ft*.fits
for telem in $($UTIL/any_filename telemetry); do
$UTIL/log_entry "Running frfread on $telem"
##############
# Object name
##############
object=$($UTIL/read_fits_keyword ${telem}[1] OBJECT)
$UTIL/setup_parfile ${frfread}.par frf_file=$telem \
origin="GSFC" \
object="$object" \
seqpi="$seqpi" \
ranom=$ra \
decnom=$dec \
clobber=yes
#####################################################################
# before running, reset the par file environment variables
# to the frfread directory, then set back to the FTOOLS directory
# when done
#####################################################################
export SYSPFILES="${frfread%/*}"
export PFILES="$LOCPFILES;$SYSPFILES"
$frfread >stdout.log 2>stderr.log
code=$?
export SYSPFILES="SYSPFILES_save"
export PFILES="$PFILES_save"
###########################################
# check the number of dropped superframes
###########################################
sf_processed=$($UTIL/read_parfile ${frfread##*/}.par sf_processed)
sf_total=$($UTIL/read_fits_keyword $telem[1] NAXIS2)
sf_lost=$($STOOL/equals "int(100*($sf_total-$sf_processed)/$sf_total+.5)")
##############################
# debugging output
#############################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: sf_processed=$sf_processed
echo ${0##*/}: sf_total=$sf_total
echo ${0##*/}: sf_lost=$sf_lost
fi
if [ $sf_lost -gt $lost_sf_percent ]; then
$UTIL/exception $0 2 "$sf_lost% of super frames in $telem corrupted"
else
$UTIL/log_entry "$sf_lost% of superframes in $telem corrupted"
fi
grep -v "seconds behind" stderr.log >filtered.log
$UTIL/ftool_test ${frfread##*/} $code $0 2 stdout.log stderr.log \
filtered.log
done
################################
# Remove files with zero events
################################
rm -f list.tmp
ls ft*.fits 2>/dev/null | grep -v HK | awk '{print $1"[0]" }' >list.tmp
rm -f cat.tmp
while [ -s list.tmp ]; do
##############################################
# Create filecat in chunks of 900 files since
# ffilecat can only handle 999 at a time
##############################################
rm -f sublist.tmp
head -900 list.tmp >sublist.tmp
tail +901 list.tmp >dum.tmp
mv -f dum.tmp list.tmp
$UTIL/setup_parfile $FTOOL/ffilecat.par infile=@sublist.tmp \
outfile=cat.tmp \
keywords=NEVENTS
$FTOOL/ffilecat >stdout.log 2>stderr.log
$UTIL/ftool_test ffilecat $? $0 2 stdout.log stderr.log
done
rm -f list.tmp
rm -f sublist.tmp
##############################
# Select files with NEVENTS=0
##############################
rm -f cat2.tmp
$UTIL/setup_parfile $FTOOL/fselect.par infile=cat.tmp \
outfile=cat2.tmp \
expr="NEVENTS == 0" \
histkw=no \
copyall=no \
keycopy=no \
clobber=yes
$FTOOL/fselect >stdout.log 2>stderr.log
$UTIL/ftool_test fselect $? $0 2 stdout.log stderr.log
rm -f cat.tmp
##########################
# Remove all listed files
##########################
$UTIL/dump_table cat2.tmp FILENAME >cat3.tmp
rm -f cat2.tmp
$UTIL/log_entry "Removing the following files with NEVENTS=0"
$UTIL/file_to_log cat3.tmp
for file in $(cat cat3.tmp); do
rm -f ${file%\[*}
done
rm -f cat3.tmp
#########################################
# Remove files with empty GTI extensions
#########################################
$UTIL/log_entry "Checking for empty GTI extensions"
rm -f list.tmp
ls ft*.fits 2>/dev/null | grep -v HK | awk '{print $1"[2]" }' >list.tmp
rm -f cat.tmp
while [ -s list.tmp ]; do
##############################################
# Create filecat in chunks of 900 files since
# ffilecat can only handle 999 at a time
##############################################
rm -f sublist.tmp
head -900 list.tmp >sublist.tmp
tail +901 list.tmp >dum.tmp
mv -f dum.tmp list.tmp
$UTIL/setup_parfile $FTOOL/ffilecat.par infile=@sublist.tmp \
outfile=cat.tmp \
keywords=NAXIS2
$FTOOL/ffilecat >stdout.log 2>stderr.log
$UTIL/ftool_test ffilecat $? $0 2 stdout.log stderr.log
done
rm -f list.tmp
rm -f sublist.tmp
#############################
# Select files with NAXIS2=0
#############################
rm -f cat2.tmp
$UTIL/setup_parfile $FTOOL/fselect.par infile=cat.tmp \
outfile=cat2.tmp \
expr="NAXIS2 == 0" \
histkw=no \
copyall=no \
keycopy=no \
clobber=yes
$FTOOL/fselect >stdout.log 2>stderr.log
$UTIL/ftool_test fselect $? $0 2 stdout.log stderr.log
rm -f cat.tmp
##########################
# Remove all listed files
##########################
$UTIL/dump_table cat2.tmp FILENAME >cat3.tmp
rm -f cat2.tmp
if [ $(wc -w <cat3.tmp) -gt 0 ]; then
$UTIL/exception $0 1 "Removing files with empty GTI extensions"
$UTIL/file_to_log cat3.tmp
for file in $(cat cat3.tmp); do
rm -f ${file%\[*}
done
fi
rm -f cat3.tmp
########################################
# Create ALLGTI extensions for HK files
########################################
$UTIL/log_entry "Appending ALLGTI extensions to housekeeping files"
for inst in S0 S1 G2 G3; do
###########################################
# List all event files for that instrument
###########################################
rm -f list.tmp
rm -f z.tmp
rm -f allgti.tmp
ls ft*${inst}*.fits 2>/dev/null | grep -v HK | \
awk '{print $1"[2]"}' >list.tmp
if [ ! -s list.tmp ]; then
rm -f list.tmp
$UTIL/log_entry "No $inst event files, so skipping ALLGTI"
continue
fi
#################
# Merge the GTIs
#################
$UTIL/gti_merge z.tmp list.tmp
rm -f list.tmp
################################
# Remove zero and negative GTIs
################################
$UTIL/setup_parfile $FTOOL/fselect.par infile=z.tmp \
outfile=allgti.tmp \
expr="STOP>START" \
copyall=no \
keycopy=yes
$FTOOL/fselect >stdout.log 2>stderr.log
$UTIL/ftool_test fselect $? $0 2 stdout.log stderr.log
##############################
# call the extension "ALLGTI"
##############################
$UTIL/add_fits_keyword allgti.tmp[1] EXTNAME ALLGTI
#####################
# debugging output
#####################
if [ -n "$DEBUG" ]; then
echo "${0##*/}: about to append allgti extension"
fi
if [ -a allgti.tmp ]; then
##########################################
# Append the GTI extension to the HK file
##########################################
list=$(ls ft*${inst}HK.fits 2>/dev/null)
for house in $list; do
###################################################
# Make sure only GTIs between the TSTART and TSTOP
# of the HK file are included
###################################################
tstart=$($UTIL/read_fits_keyword $house[0] TSTART)
tstop=$( $UTIL/read_fits_keyword $house[0] TSTOP )
#####################
# debugging output
#####################
if [ -n "$DEBUG" ]; then
echo "${0##*/}: house=$house"
echo "${0##*/}: tstart=$tstart"
echo "${0##*/}: tstop=$tstop"
fi
rm -f gti.tmp
$UTIL/setup_parfile $FTOOL/fselect.par \
infile=allgti.tmp \
outfile=gti.tmp \
expr="STOP>= $tstart && START <=$tstop"
$FTOOL/fselect >stdout.log 2>stderr.log
$UTIL/ftool_test fselect $? $0 2 stdout.log stderr.log
##############################################
# Transfer keyword values from primary header
##############################################
for key in TSTART TSTOP TELAPSE DATAMODE \
DATE-OBS TIME-OBS DATE-END TIME-END; do
value=$($UTIL/read_fits_keyword $house[0] $key)
#####################
# debugging output
#####################
if [ -n "$DEBUG" ]; then
echo "${0##*/}: key=$key"
echo "${0##*/}: value=$value"
fi
$UTIL/add_fits_keyword gti.tmp[1] $key $value
done
###########################
# Append the GTI extension
###########################
$UTIL/setup_parfile $FTOOL/fappend.par infile=gti.tmp \
outfile=$house
$FTOOL/fappend >stdout.log 2>stderr.log
$UTIL/ftool_test fappend $? $0 2 stdout.log stderr.log
###########################################
# Set the ONTIME keyword in all extensions
###########################################
ontime=$($STOOL/compute_exposure $house)
for ext in 0 1 2; do
$UTIL/add_fits_keyword ${house}[${ext}] ONTIME $ontime
done
done
fi
rm -f allgti.tmp
done
exit 0