wrapup (version 2.0)
You can also look at:
#! /usr/bin/ksh
###########################################################################
#
# SYNTAX: wrapup
#
# BRIEFLY: Do a final wrap-up of all files.
#
# DESCRIPTION: This routine does some final manipulation of the files.
# DESCRIPTION: First, add a set of FITS keywords describing the processing
# DESCRIPTION: and observation to each FITS file.
# DESCRIPTION: Then run the <TT>fverify</TT> FTOOL on all FITS files
# DESCRIPTION: and report any errors found.
# DESCRIPTION: Next, check that all files in either the distribution or the
# DESCRIPTION: trend catalogs are present. Report an error if a file is missing
# DESCRIPTION: or if there are any files not listed in either catalog.
# DESCRIPTION: Finally, save the names of a few important files in the job
# DESCRIPTION: parameter file so that they may be used by software downstream
# DESCRIPTION: in the processing pipeline.
#
# VERSION: 2.0
#
# HISTORY: 0.0 -> 1.0 8/21/96
# HISTORY: Now removes z.tmp temporary file.
# HISTORY:
# HISTORY: 1.0 -> 1.1 9/23/96
# HISTORY: Now sets world read permission for .html amd .par files.
# HISTORY:
# HISTORY: 1.1 -> 1.2 12/26/96
# HISTORY: Now records tapecat, trendcat, and header file names in the job.par.
# HISTORY: 1/2/97 Also record filenames of front and back cover images.
# HISTORY:
# HISTORY: 1.2 -> 1.3 2/7/97
# HISTORY: Corrected the typo "usppiname", which caused the USPINAME keywords
# HISTORY: to be blank. Also supplied missing "$" for jppinum != 0 test.
# HISTORY:
# HISTORY: 1.3 -> 1.4 3/3/97
# HISTORY: Now re-saves the jobpar at the end of this routine. Previously
# HISTORY: the job par was saved in $SUBS/filelists. It is still saved there
# HISTORY: so that it can be included in the list, but it is re-saved here
# HISTORY: so that keywords added after $SUBS/filelists are also saved.
# HISTORY:
# HISTORY: 1.4 -> 1.5 5/23/97
# HISTORY: Now chooses the highest resolution (last listed) GIS source
# HISTORY: picture for the back cover of the CD.
# HISTORY:
# HISTORY: 1.5 -> 1.6 6/19/97
# HISTORY: Deleted the code which set world read permission on some files.
# HISTORY: That permission is clobbered anyway when the files are copied.
# HISTORY:
# HISTORY: 1.6 -> 1.7 10/11/97
# HISTORY: Added stdout.top.log and stderr.top.log to the list of files
# HISTORY: which are not considered "leftovers". Also changed the error
# HISTORY: message for leftover files so that it does not say it is removing
# HISTORY: the file.
# HISTORY:
# HISTORY: 1.7 -> 1.8 12/29/97
# HISTORY: Fixed the syntax in several exception calls.
# HISTORY:
# HISTORY: 1.8 -> 1.9 1999-07-01
# HISTORY: Changed stdout.top.log and stderr.top.log to
# HISTORY: wrapup.stdout.log and wrapup.stderr.log to accomodate the
# HISTORY: use of $UTIL/run_subroutine
# HISTORY:
# HISTORY: 1.9 -> 2.0 1999-10-14
# HISTORY: Modified to check the numerrs parameter from the new fverify
#
############################################################################
#DEBUG=1
$UTIL/milestone "Doing final wrap up of all files"
######################################################
# Dump the filenames in the tape.cat to an ascii file
######################################################
dump=dump.tmp
tapecat=$($UTIL/generate_filename tapecat)
if [ ! -a $tapecat ]; then
$UTIL/exception $0 3 "No tapecat file: $tapecat"
fi
###################
# Debugging output
###################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: tapecat=$tapecat
fi
rm -f $dump
$UTIL/setup_parfile $FTOOL/fdump.par infile=$tapecat \
outfile=$dump \
rows=- \
columns="filename" \
prhead=no \
prdata=yes \
showcol=no \
showunit=no \
showscale=no \
showrow=no
$FTOOL/fdump >stdout.log 2>stderr.log
$UTIL/ftool_test fdump $? $0 2 stdout.log stderr.log
#######################
# Adding fits keywords
#######################
$UTIL/log_entry "Adding FITS keywords to all distributed FITS files"
#######################################################################
# Assemble what you will need to know
# fmodhead.tmp contains the information about keywords
# which are the same for all fits files.
# Later z.tmp will contain this info plus the info unique to each file
# such as the file name
#######################################################################
rm -f fmodhead.tmp
sequence=$($UTIL/read_parfile $JOBPAR sequence)
seqpnum=$( $UTIL/read_parfile $JOBPAR seqprocnum)
procver=$( $UTIL/read_parfile $PARFILE version)
echo "SEQNUM $sequence / Sequential number from ODB" >>fmodhead.tmp
echo "SEQPNUM $seqpnum / Number of times sequence processed" >>fmodhead.tmp
echo "PROCVER $procver / Processing configuration number" >>fmodhead.tmp
uspinum=$($UTIL/read_parfile $JOBPAR uspinum)
if [ "$uspinum" != "0" ]; then
uspiname=$($UTIL/read_parfile $JOBPAR uspiname)
uspiadr1=$($UTIL/read_parfile $JOBPAR uspiaddr1)
uspiadr2=$($UTIL/read_parfile $JOBPAR uspiaddr2)
uspiadr3=$($UTIL/read_parfile $JOBPAR uspiaddr3)
uspiadr4=$($UTIL/read_parfile $JOBPAR uspiaddr4)
uspiadr5=$($UTIL/read_parfile $JOBPAR uspiaddr5)
uspieadr=$($UTIL/read_parfile $JOBPAR uspiemail)
echo "USPINUM $uspinum" >> fmodhead.tmp
echo "USPINAME '$uspiname'" >> fmodhead.tmp
echo "USPIADR1 '$uspiadr1'" >> fmodhead.tmp
echo "USPIADR2 '$uspiadr2'" >> fmodhead.tmp
echo "USPIADR3 '$uspiadr3'" >> fmodhead.tmp
echo "USPIADR4 '$uspiadr4'" >> fmodhead.tmp
echo "USPIADR5 '$uspiadr5'" >> fmodhead.tmp
echo "USPIEADR '$uspieadr'" >> fmodhead.tmp
fi
jppinum=$($UTIL/read_parfile $JOBPAR jppinum)
if [ "$jppinum" != "0" ]; then
jppiadr1=$($UTIL/read_parfile $JOBPAR jppiaddr1)
jppiadr2=$($UTIL/read_parfile $JOBPAR jppiaddr2)
jppiadr3=$($UTIL/read_parfile $JOBPAR jppiaddr3)
jppiadr4=$($UTIL/read_parfile $JOBPAR jppiaddr4)
jppiadr5=$($UTIL/read_parfile $JOBPAR jppiaddr5)
jppieadr=$($UTIL/read_parfile $JOBPAR jppiemail)
echo "JPPINUM $jppinum" >> fmodhead.tmp
echo "JPPINAME '$jppiname'" >> fmodhead.tmp
echo "JPPIADR1 '$jppiadr1'" >> fmodhead.tmp
echo "JPPIADR2 '$jppiadr2'" >> fmodhead.tmp
echo "JPPIADR3 '$jppiadr3'" >> fmodhead.tmp
echo "JPPIADR4 '$jppiadr4'" >> fmodhead.tmp
echo "JPPIADR5 '$jppiadr5'" >> fmodhead.tmp
echo "JPPIEADR '$jppieadr'" >> fmodhead.tmp
fi
##################################
# Loop through all the FITS files
##################################
for file in $(grep -v \.wrap $dump); do
###################
# Debugging output
###################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: file=$file
fi
if [ -a "$file" ]; then
###############################
# Add info unique to this file
###############################
rm -f z.tmp
cp fmodhead.tmp z.tmp
echo "FNAME '$file'" >>z.tmp
$UTIL/setup_parfile $FTOOL/fmodhead.par tmpfil=z.tmp \
infile="${file}[0]"
$FTOOL/fmodhead >stdout.log 2>stderr.log
$UTIL/ftool_test fmodhead $? $0 2 stdout.log stderr.log
rm -f z.tmp
fi
done
rm -f fmodhead.tmp
################################
# Run fverify on all fits files
################################
$UTIL/log_entry "Running fverify on all distributed files"
for file in $(grep -v \.wrap $dump); do
###################
# Debugging output
###################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: file=$file
fi
if [ -a "$file" ]; then
$UTIL/setup_parfile $FTOOL/fverify.par prhead=no \
testdata=no \
infile=$file
$FTOOL/fverify >stdout.log 2>stderr.log
code=$?
ftoolsversion=$(echo $FTOOL | sed -e's|/|\
|g' | grep ftools\.)
ftoolsversion=${ftoolsversion#ftools}
ftoolsversion=${ftoolsversion#.}
if [ -n "$DEBUG" ]; then
echo "${0##*/}: ftoolsversion=$ftoolsversion"
fi
if [ -n "$new_fverify" ]; then
numerrs=$($UTIL/read_parfile fverify.par numerrs)
else
numerrs=0
fi
if [ $? -ne 0 -o -s stderr.log -o $numerrs -gt 0 ]; then
####################
# There is an error
####################
$UTIL/exception $0 2 "$file failed fverify"
$UTIL/log_entry "stdout output from fverify:"
$UTIL/file_to_log stdout.log
$UTIL/log_entry "stderr output from fverify"
$UTIL/file_to_log stderr.log
fi
rm -f stdout.log
rm -f stderr.log
rm -f fverify.par
fi
done
#########################################
# Calculate checksums for all fits files
#########################################
$UTIL/log_entry "Calculating checksums for all FITS files"
for file in $(grep -v \.wrap $dump); do
if [ -a "$file" ]; then
$UTIL/setup_parfile $FTOOL/fchecksum.par infile=$file \
update=yes \
datasum=yes
$FTOOL/fchecksum >stdout.log 2>stderr.log
$UTIL/ftool_test fchecksum $? $0 2 stdout.log stderr.log
fi
done
########################################################
# Clear out any files not in the tape cat or trend cat.
# Don't delete job.par or process.par
########################################################
$UTIL/log_entry "Doing inventory of all files"
######################################################
# Dump the trend catalog and add trend files to $dump
######################################################
trendcat=$($UTIL/generate_filename trendcat)
if [ ! -a $trendcat ]; then
$UTIL/exception $0 3 "No trendcat file: $trendcat"
fi
rm -f trend.tmp
$UTIL/setup_parfile $FTOOL/fdump.par infile=$trendcat \
outfile=trend.tmp \
rows=- \
columns="filename" \
prhead=no \
prdata=yes \
showcol=no \
showunit=no \
showscale=no \
showrow=no
$FTOOL/fdump >stdout.log 2>stderr.log
$UTIL/ftool_test fdump $? $0 2 stdout.log stderr.log
cat trend.tmp >>$dump
rm -f trend.tmp
#######################################################
# Add a couple extra files which shouldn't be removed,
# but aren't included in the tape and trend catalogs
#######################################################
echo "$JOBPAR" >>$dump
echo "$PARFILE" >>$dump
echo "$dump" >>$dump
echo "wrapup.stdout.log" >>$dump
echo "wrapup.stderr.log" >>$dump
##########################
# Remove all excess files
##########################
for file in $( ls -a ); do
if [ -z "$(grep $file $dump)" ]; then
$UTIL/exception $0 1 "Leftover file: $file"
fi
done
##########################
# Check for missing files
##########################
for file in $( cat $dump ); do
file=${file%.wrap}
if [ ! -a "$file" ]; then
$UTIL/exception $0 2 "$file is missing"
fi
done
rm -f $dump
##################################################
# Record some important file names in the job.par
##################################################
tapecat=$( $UTIL/generate_filename tapecat )
trendcat=$($UTIL/generate_filename trendcat)
header=$( $UTIL/generate_filename header )
cover1=$( $UTIL/generate_filename sourcepic sis any any any)
cover2=$( $UTIL/generate_filename sourcepic gis any any any)
cover1=$(ls $cover1 2>/dev/null | tail -1)
cover2=$(ls $cover2 2>/dev/null | tail -1)
if [ -z "$cover1" ]; then
cover1=$cover2
fi
if [ -z "$cover2" ]; then
cover2=$cover1
fi
$UTIL/add_parameter $JOBPAR tapecat $tapecat s "Distribution catalog file"
$UTIL/add_parameter $JOBPAR trendcat $trendcat s "Trend product catalog file"
$UTIL/add_parameter $JOBPAR header $header s "HTML header file"
$UTIL/add_parameter $JOBPAR cover1 "$cover1" s "CD front cover image"
$UTIL/add_parameter $JOBPAR cover2 "$cover2" s "CD back cover image"
##################
# Save the jobpar
##################
jobpar=$( $UTIL/generate_filename jobpar)
cp $JOBPAR $jobpar
exit 0