screen (version 4.2)
You can also look at:
#! /usr/bin/ksh
###########################################################################
#
# SYNTAX: screen
#
# BRIEFLY: Run xselect to screen out bad data.
#
# DESCRIPTION: This routine runs xselect on the unfiltered event files
# DESCRIPTION: to produce the filtered event files.
# DESCRIPTION: The following screening criteria are applied:
# DESCRIPTION: <H3>Filtering Criteria for Both Instruments:</H3>
# DESCRIPTION: <UL>
# DESCRIPTION: <LI> SAA==0 (not in South Atlantic Anomaly)
# DESCRIPTION: <LI> ELV > %elv (sufficiently far from limb of Earth)
# DESCRIPTION: <LI> ACS==0 (Attitude control in fine guidance mode)
# DESCRIPTION: <LI> ANG_DIST > 0 && ANG_DIST < %ang_dist
# DESCRIPTION: (Angular distance from the nominal pointing is
# DESCRIPTION: below threshold)
# DESCRIPTION: </UL>
# DESCRIPTION: <H3>Additional SIS Filtering Criteria:</H3>
# DESCRIPTION: <UL>
# DESCRIPTION: <LI> COR > %cor (Rigidity of Earth's magnetic field high
# DESCRIPTION: enough to shield the instruments from particles)
# DESCRIPTION: <LI> FOV==0 (Telescope is pointing at the sky)
# DESCRIPTION: <LI> BR_EARTH > %br_earth (far enough from the limb of the bright
# DESCRIPTION: Earth so that light leaks will not affect the CCDs)
# DESCRIPTION: <LI> Sn_SATFm != 1 (Telemetry is not saturated )
# DESCRIPTION: <LI> T_DY_NT <0 || T_DY_NT > %delayperccd per CCD
# DESCRIPTION: (The time since the last day-night transition
# DESCRIPTION: above threshold)
# DESCRIPTION: <LI> T_SAA < 0 || T_SAA > %delayperccd per CCD (The time
# DESCRIPTION: since the last exit from the
# DESCRIPTION: South Atlantic Anomaly
# DESCRIPTION: above threshold)
# DESCRIPTION: <LI> Sn_PIXLm is within %pixl_nsigma of its mean value.
# DESCRIPTION: The details of this screening criterion are described in
# DESCRIPTION: $UTIL/pixl_criteria.
# DESCRIPTION: <LI> Only grade 0, 2, 3, and 4 events are extracted.
# DESCRIPTION: <LI> The data are cleaned with sisclean which removes hot and
# DESCRIPTION: flickering pixels.
# DESCRIPTION: </UL>
# DESCRIPTION: <H3>Additional GIS Cleaning Criteria:</H3>
# DESCRIPTION: <UL>
# DESCRIPTION: <LI> COR>4 (Rigidity of Earth's magnetic field high enough
# DESCRIPTION: to shield the instruments from particles)
# DESCRIPTION: <LI> G2_H0+G2_H2+G3_H0+G3_H2 < 45
# DESCRIPTION: <LI> G2_H0+G2_H2+G3_H0+G3_H2 < 0.45*COR**2-13*COR+125
# DESCRIPTION: <LI> RBM_CONT < 100
# DESCRIPTION: <LI> Gn_L1 > 0 (The L1 count rate is above zero)
# DESCRIPTION: <LI> A region filter is applied to GIS data to filter
# DESCRIPTION: out the noisy edges of the detectors and the calibration
# DESCRIPTION: source.
# DESCRIPTION: <LI> The data are cleaned with gisclean which applies a
# DESCRIPTION: PI - RTI rejection criterion
# DESCRIPTION: </UL>
# DESCRIPTION: <P>
# DESCRIPTION: Note that the exact screening criteria applied to each event
# DESCRIPTION: file are recorded in the processing log.
#
# VERSION: 4.2
#
# HISTORY: 0.0 -> 1.0 8/20/96
# HISTORY: Now skips FAST mode files.
# HISTORY:
# HISTORY: 1.0 -> 1.1 9/11/96
# HISTORY: Now deletes filtered event files with zero events.
# HISTORY:
# HISTORY: 1.1 -> 1.2 9/27/96
# HISTORY: Left out a $UTIL when making the last change.
# HISTORY: 9/30/96
# HISTORY: Also added a check to be sure the file existed before trying
# HISTORY: NEVENTS keyword, and specified primary extension to read
# HISTORY: keyword from.
# HISTORY:
# HISTORY: 1.2 -> 1.3 11/25/96
# HISTORY: Now extracts bright and dark Earth event files and GIS calibration
# HISTORY: source spectra. Also moved the xselect clean commands to appear
# HISTORY: before the "select mkf" command, to accomodate the different filters.
# HISTORY:
# HISTORY: 1.3 -> 1.4 12/23/96
# HISTORY: Now skips region filtering for MPC mode data.
# HISTORY: 1/3/97 No longer extract SIS bright Earth data.
# HISTORY:
# HISTORY: 1.4 -> 1.5 2/24/97
# HISTORY: Removed code which extracts calsource spectra, since these are
# HISTORY: now generated in the calsource routine.
# HISTORY:
# HISTORY: 1.5 -> 1.6 4/16/97
# HISTORY: Stopped skipping FAST mode files.
# HISTORY: Took out bright and dark Earth extractions, and moved them to
# HISTORY: a separate module.
# HISTORY: Now uses different GIS regions, depending on whether the
# HISTORY: spread discriminator is ON or OFF.
# HISTORY:
# HISTORY: 1.6 -> 2.0 5/14/97
# HISTORY: Now applies rev2 screening criteria for GIS data.
# HISTORY: Went back to skipping MPC mode files, since you can't extract events,
# HISTORY: at least with the current xselect.
# HISTORY: Added grade selection (grades 0,2,3,4).
# HISTORY:
# HISTORY: 2.0 -> 2.1 5/14/97
# HISTORY: Changed mkf_def_expr to ASCA_mkf_def_expr in xselect. Apparently
# HISTORY: the advent of ftools 4.0 made "mkf_def_expr" an error, which
# HISTORY: caused xselect to not do any GTI selection at all, which is of course
# HISTORY: disastrous.
# HISTORY:
# HISTORY: 2.1 -> 3.0 7/7/97
# HISTORY: Moved cleaning to occur after filtering. For the SIS this is the
# HISTORY: correct way to do things, because some of the spurious, filtered out
# HISTORY: events may have been causing cleansis to mark out some pixels which
# HISTORY: weren't really bad. For some reason things had always been done
# HISTORY: in the other order in the past, but that was incorrect.
# HISTORY: 7/15/97
# HISTORY: Added an explicit check that NAXIS2 = NEVENTS in the filtered files.
# HISTORY: This is to compensate for a bug in gisclean which does not do this,
# HISTORY: at least in FTOOLS 4.0. The problem became apparent when the gisclean
# HISTORY: run was moved to occur after the extractor run.
# HISTORY:
# HISTORY: 3.0 -> 3.1 8/4/97
# HISTORY: Now does not run gisclean if RISEBINS=1. This can occur when using
# HISTORY: non-standard bit assignments.
# HISTORY: A mix of cleaned and uncleaned files could cause problems downstream,
# HISTORY: especially for spectra, but the splitting on RAW_X_BINS and
# HISTORY: PHA_BINS should avoid this.
# HISTORY: 8/5/97 Made some fixes for FAST mode. Specifically supplied a
# HISTORY: (yes) to the question "Switch from PI to PHA?", since FAST mode
# HISTORY: files do not have a PI column. Also changed grade selection
# HISTORY: for FAST mode to only select grade 0.
# HISTORY:
# HISTORY: 3.1 -> 3.2 8/19/97
# HISTORY: Removed BR_EARTH criterion from GIS data.
# HISTORY:
# HISTORY: 3.2 -> 4.0 2/20/98
# HISTORY: Began using the "sigma-based" screening criteria for Sn_PIXLm.
# HISTORY:
# HISTORY: 4.0 -> 4.1 4/10/98
# HISTORY: Deleted the ASCA_mkf_def_expr parameter in the select mkf command
# HISTORY: in the xselect command files. This is to accomodate
# HISTORY: FTOOLS 4.1. The parameter name has changed and using the old
# HISTORY: parameter name causes xselect to not do any time filtering.
# HISTORY: Also added a "save events" command for FAST mode data. This
# HISTORY: had been left out so that screened FAST mode event file
# HISTORY: were not being produced.
# HISTORY: Also removed the "yes" in response to the "switch to PHA"
# HISTORY: message from xselect. This is no longer needed in FTOOLS 4.1
# HISTORY:
# HISTORY: 4.1 -> 4.2 2000-03-01
# HISTORY: Now explicitly set coordinates to detector for xselect.
###########################################################################
#DEBUG=1
command=command.tmp
criteria=criteria.tmp
$UTIL/milestone "Cleaning and filtering the unfiltered event files"
for inst in s0 s1 g2 g3; do
#########################
# Criteria from par file
#########################
br_earth=$( $UTIL/read_parfile $PARFILE br_earth )
elv=$( $UTIL/read_parfile $PARFILE elv )
cor=$( $UTIL/read_parfile $PARFILE cor )
ang_dist=$( $UTIL/read_parfile $PARFILE ang_dist )
delayperccd=$($UTIL/read_parfile $PARFILE delayperccd)
###################
# Debugging output
###################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: inst=$inst
fi
####################################
# Loop through the unfiltered files
####################################
list=$($UTIL/generate_filename unfiltered $inst any any any any)
list=$(ls $list 2>/dev/null)
for unfiltered in $list; do
index=$( $UTIL/parse_filename index $unfiltered)
mode=$( $UTIL/parse_filename mode $unfiltered)
bitrate=$($UTIL/parse_filename bitrate $unfiltered)
##################################
# Skip SIS FAINT & MPC mode files
##################################
if [ "$mode" = "01" -o "$mode" = "71" ]; then
$UTIL/log_entry "Skipping $unfiltered because of mode"
continue
fi
filtered=$($UTIL/generate_filename event \
$inst $index $mode $bitrate)
rm -f $filtered
$UTIL/log_entry "Filtering $unfiltered into $filtered"
###################
# Debugging output
###################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: inst=$inst
echo ${0##*/}: index=$index
echo ${0##*/}: mode=$mode
echo ${0##*/}: bitrate=$bitrate
echo ${0##*/}: unfiltered=$unfiltered
echo ${0##*/}: filtered=$filtered
fi
####################################
####################################
##
## Create an XSELECT command file
##
####################################
####################################
#########################################
# Start xselect and read unfiltered file
#########################################
rm -f $command
echo "screen" >>$command
echo "set mission ASCA" >>$command
echo "set datadir ." >>$command
echo "read events $unfiltered" >>$command
#############################################
# Standard screening criteria and "cleaning"
#############################################
case "$inst" in
s?)
######
# SIS
######
rm -f $criteria
echo "(SAA==0)" >> $criteria
echo "&&(BR_EARTH>${br_earth})" >> $criteria
echo "&&(ELV>${elv})" >> $criteria
echo "&&(COR>${cor})" >> $criteria
echo "&&(ACS==0)" >> $criteria
echo "&&(ANG_DIST>0)&& (ANG_DIST<${ang_dist})" >> $criteria
echo "&&(FOV==0)" >> $criteria
ccdlst=$( $UTIL/read_fits_keyword $unfiltered[0] \
S${inst#s}CCDLST)
ccdmode=$($UTIL/read_fits_keyword $unfiltered[0] \
S${inst#s}CCDMOD)
echo "&&" >> $criteria
$UTIL/pixl_criteria $unfiltered >> $criteria
for ccd in $ccdlst ; do
echo "&&( (S${inst#s}_SATF${ccd}<1)" >> $criteria
echo " ||(S${inst#s}_SATF${ccd}>1))" >> $criteria
done
time=$(expr $ccdmode \* $delayperccd)
echo "&&((T_DY_NT<0)||(T_DY_NT>${time}))" >> $criteria
echo "&&((T_SAA <0)||(T_SAA >${time}))" >> $criteria
##################
# Grade selection
##################
if [ "$mode" != "03" ]; then
#################################
# Regular BRIGHT or BRIGHT2 mode
#################################
echo "select event" \
"\"grade==0||grade==2||grade==3||grade==4\" " \
"save_file=no" >>$command
else
##########################################################
# In FAST mode grade 0 means good and grade 1 means noise
##########################################################
echo "select event \"grade==0\" " >>$command
fi
###################
# Debugging output
###################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: ccdlst=$ccdlst
echo ${0##*/}: ccdmode=$ccdmode
echo ${0##*/}: time=$time
fi
;;
g?)
######
# GIS
######
rm -f $criteria
echo "(SAA==0)&&" >> $criteria
echo "(COR>4)&&" >> $criteria
echo "(G2_H0+G2_H2+G3_H0+G3_H2<45)&&" >> $criteria
echo "(G2_H0+G2_H2+G3_H0+G3_H2<0.45*COR**2-13*COR+125)&&"\
>> $criteria
echo "(RBM_CONT<100)&&" >> $criteria
echo "(ELV>5)&&" >> $criteria
echo "(ACS==0)&&" >> $criteria
echo "(ANG_DIST>0)&& (ANG_DIST<${ang_dist})&&" >> $criteria
echo "(G${inst#g}_L1>0)" >> $criteria
###########################################
# Info needed for selecting region filters
###########################################
if [ "$mode" != "71" ]; then
##################################
# No region filtering in MPC mode
##################################
dir=$($UTIL/read_parfile $PARFILE regiondir)
rawxbins=$($UTIL/read_fits_keyword $unfiltered RAWXBINS)
spread=$( $UTIL/read_fits_keyword $unfiltered[0] S_DSCR)
#######################
# Usable field of view
#######################
region=$($UTIL/read_parfile $PARFILE \
${inst}region$rawxbins$spread)
$UTIL/fetch_file $dir $region
###################
# Debugging output
###################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: dir=$dir
echo ${0##*/}: rawxbins=$rawxbins
echo ${0##*/}: spread=$spread
echo ${0##*/}: region=$region
fi
echo "set image det" >>$command
echo "filter region $region" >>$command
fi
;;
*)
#####################
# Unknown instrument
#####################
$UTIL/exception $0 1 "Unknown instrument $inst";;
esac
############################
# Extract the filtered file
############################
echo "select mkf \"@${criteria}\" data_dir=. " \
>>$command
echo "extract events" >>$command
########
# Clean
########
case "$inst" in
s?)
######
# SIS
######
if [ "$mode" != "03" ]; then
##################################
# Don't run sisclean in fast mode
##################################
echo "sisclean saoimage=no sis_plot=no clean=2 cellsize=5" \
"log_prob=-5.24 bkg_thr=3 clean_phalow=0 clean_phahi=4095" \
>> $command
echo "save clean outfile=$filtered use_events=no" >>$command
else
####################################
# save the extracted fast mode file
####################################
echo "save events outfile=$filtered use_events=no" >>$command
fi
;;
g?)
######
# GIS
######
risebins=$($UTIL/read_fits_keyword $unfiltered[0] RISEBINS)
if [ $risebins -gt 1 ]; then
###########################################
# There is rise time info, so run gisclean
###########################################
echo "gisclean" >>$command
echo "save clean outfile=$filtered use_events=no" >>$command
else
###########################################
# No rise time info, so don't run gisclean
###########################################
$UTIL/log_entry "Skipping gisclean since RISEBINS=$risebins"
echo "save events outfile=$filtered use_events=no">>$command
fi
;;
esac
###############
# Exit xselect
###############
echo "exit save_session=no" >>$command
###################
# Debugging output
###################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: filtered=$filtered
echo ${0##*/}: dark=$dark
echo ${0##*/}: bright=$bright
fi
$UTIL/log_entry "Using the following selection criteria:"
$UTIL/file_to_log criteria.tmp
##############
# Run xselect
##############
$FTOOL/xselect @${command} </dev/null > stdout.log 2> stderr.log
$UTIL/xselect_test $? $0 stdout.log stderr.log
rm -f $criteria
rm -f $command
if [ -a "$filtered" ]; then
nevents=$($UTIL/read_fits_keyword ${filtered}[0] NEVENTS)
##############################################################
# Make sure NEVENTS = NAXIS2. This is to work-around a bug in
# gisclean which does not update NEVENTS (ftools 4.0)
##############################################################
naxis2=$($UTIL/read_fits_keyword ${filtered}[1] NAXIS2)
if [ $nevents -ne $naxis2 ]; then
##############################
# Need to fix NEVENTS keyword
##############################
nevents=$naxis2
$UTIL/log_entry "Setting NEVENTS to $naxis2 in $unfiltered"
$UTIL/add_fits_keyword ${filtered}[0] NEVENTS $naxis2
fi
#######################################
# Remove filtered files with no events
#######################################
if [ $nevents -eq 0 ]; then
$UTIL/log_entry \
"Deleting $filtered since it contains $nevents events"
rm -f $filtered
fi
fi
done
done
exit 0