filelists (version 2.1)
You can also look at:
#! /usr/bin/ksh
###########################################################################
#
# SYNTAX: filelists
#
# BRIEFLY: Make lists of the files to be distributed.
#
# DESCRIPTION: This routine lists the files which are to be distributed.
# DESCRIPTION: This is done in two ways:
# DESCRIPTION: <ol>
# DESCRIPTION: <li> The FITS tape catalog
# DESCRIPTION: <li> An HTML file
# DESCRIPTION: </ol>
#
# VERSION: 2.1
#
# HISTORY: 0.0 -> 1.0 8/19/96
# HISTORY: Removed the (.wrap) after wrapped file names
# HISTORY: in the HTML list (confusing for CD distributions).
# HISTORY: teldef now refers to both SIS and GIS files, so removed
# HISTORY: individual g2onflf, etc types.
# HISTORY:
# HISTORY: 1.0 -> 1.1 9/26/96
# HISTORY: Now reads list of file types to distribute from $LISTS/file_classes
# HISTORY:
# HISTORY: 1.1 -> 1.2 11/26/96
# HISTORY: Added code to create the trendcat file. This code was moved from
# HISTORY: rev1trend
# HISTORY:
# HISTORY: 1.2 -> 2.0 1/2/97
# HISTORY: re-written so that the tapecat is generated first, and then the
# HISTORY: HTML file list is created from it. The new HTML page requires that
# HISTORY: the file classes be sorted in $LISTS/file_classes
# HISTORY:
# HISTORY: 2.0 -> 2.1 3/3/97
# HISTORY: Removed a problem where files with the same description were
# HISTORY: gettinglisted separately in the HTML list by removing
# HISTORY: whitespace around the description
#
##############################################################################
#DEBUG=1
columns=columns.tmp
data=data.tmp
rm -f $data
rm -f $columns
$UTIL/milestone "Listing the files for distribution"
#########################################
# save the job.par and process.par files
#########################################
jobpar=$( $UTIL/generate_filename jobpar )
parfile=$($UTIL/generate_filename parfile )
$UTIL/log_entry "Saving $JOBPAR as $jobpar and $PARFILE as $parfile"
cp $JOBPAR $jobpar
cp $PARFILE $parfile
#################################################################
# create a dummy tape catalog file so that it can list itself
################################################################
tapecat=$( $UTIL/generate_filename tapecat )
fileinfo=$($UTIL/generate_filename fileinfo )
header=$( $UTIL/generate_filename header )
rm -f $tapecat
rm -f $fileinfo
touch $tapecat
touch $fileinfo
#######################################
# column info for tape catalog files
######################################
rm -f $columns
echo "filenum I4" >> $columns
echo "filename A57" >> $columns
echo "filesize I7 kilobytes" >> $columns
echo "fileclas A20" >> $columns
echo "descrip A64" >> $columns
################################
# list files to an ascii file
################################
typeset -i nfiles=0
typeset -i count=0
typeset -i size=0
typeset -i total_size=0
typeset -i i=1
typelist=$( grep -v "^#" $LISTS/file_classes | \
awk -F'|' '$2 !~ /none / {print $1}')
for type in $typelist; do
description=$($UTIL/file_description $type)
class=$($UTIL/file_class $type )
####################
# debugging output
####################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: type=$type
fi
####################################
# loop over the files of each type
####################################
list=$($UTIL/any_filename $type )
if [ -n "$list" ]; then
count=$(echo $list |wc -w)
let nfiles=$nfiles+$count
for file in $list; do
#################################
# get information about the file
#################################
format=$($STOOLS/isFITS $file)
if [ "$type" = "tapecat" ]; then
format=FITS
fi
size=$(wc -c $file |awk '{print $1}')
let size=$size/1000
let total_size=$total_size+$size
####################################
# entry for FITS list
####################################
if [ "$format" = "FITS" ]; then
echo "$i $file $size $class '$description' " >> $data
else
echo "$i ${file}.wrap $size $class '$description' " \
>> $data
fi
let i=$i+1
done
fi
done
let total_size=$total_size/1000
total_files=$nfiles
$UTIL/add_parameter $JOBPAR datasize "$total_size" i \
"Data set size in megabytes"
$UTIL/add_parameter $JOBPAR filecount "$total_files" i \
"Total number of distributed files"
######################################
# create the tape catalog file
######################################
$UTIL/log_entry "Creating the FITS format file catalog $tapecat"
rm -f $tapecat
$UTIL/setup_parfile $FTOOLS/fcreate.par cdfile=$columns \
datafile=$data \
outfile=$tapecat \
tbltype=ASCII
$FTOOLS/fcreate >stdout.log 2>stderr.log
$UTIL/ftool_test fcreate $? $0 2 stdout.log stderr.log
rm -f $data
rm -f $columns
############################################################################
###########################################################################
##
## catalog the trend products
##
###########################################################################
############################################################################
$UTIL/log_entry "Creating the trend product catalog"
columns=columns.tmp
data=data.tmp
rm -f $data
rm -f $columns
#################################################################
# create a dummy trend catalog file so that it can lists itself
################################################################
trendcat=$( $UTIL/generate_filename trendcat )
rm -f $trendcat
touch $trendcat
#######################################
# column info for tape catalog files
######################################
rm -f $columns
echo "filenum I4" >> $columns
echo "filename A57" >> $columns
echo "filesize I7 kilobytes" >> $columns
echo "fileclas A20" >> $columns
echo "descrip A64" >> $columns
################################
# list files to an ascii file
################################
typeset -i size=0
typeset -i i=1
typelist=$( grep -v "^#" $LISTS/file_classes | \
awk -F'|' '$3 !~ /none / {print $1}')
for type in $typelist; do
description=$($UTIL/file_description $type)
####################
# debugging output
####################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: type=$type
fi
#########################################################
# a file which goes to more than one directory has more
# than one class and is listed more than once
#########################################################
for class in $($UTIL/trend_class $type ); do
####################################
# loop over the files of each type
####################################
list=$($UTIL/any_filename $type )
if [ -n "$list" ]; then
for file in $list; do
size=$(ls -s $file |awk '{print $1}')
echo "$i $file $size $class '$description' " >> $data
let i=$i+1
done
fi
done
done
######################################
# create the trend catalog file
######################################
$UTIL/log_entry "Creating the FITS format file catalog $trendcat"
rm -f $trendcat
$UTIL/setup_parfile $FTOOLS/fcreate.par cdfile=$columns \
datafile=$data \
outfile=$trendcat \
tbltype=ASCII
$FTOOLS/fcreate >stdout.log 2>stderr.log
$UTIL/ftool_test fcreate $? $0 2 stdout.log stderr.log
rm -f $data
rm -f $columns
##############################################################################
##############################################################################
##
## Create the HTML fileinfo file from the tapecat
##
##############################################################################
##############################################################################
$UTIL/log_entry "Creating $fileinfo"
################################
# HTML header for fileinfo file
################################
seq=$($UTIL/read_parfile $JOBPAR sequence )
echo "<html>" >>$fileinfo
echo "<title>" >>$fileinfo
echo "ASCA Sequence $seq File List" >>$fileinfo
echo "</title>" >>$fileinfo
echo "<body>" >>$fileinfo
echo "<h1>ASCA Sequence $seq File List</h1>" >>$fileinfo
echo "This page lists all the files included in the" >>$fileinfo
echo "distribution for this sequence." >>$fileinfo
echo "The size of most files is given after the" >>$fileinfo
echo "file name." >>$fileinfo
echo "In total there are $total_files files" >>$fileinfo
echo "occupying $total_size megabytes of storage space." >>$fileinfo
echo "<p>" >>$fileinfo
echo "The following information is also available:" >>$fileinfo
echo "<ul>" >>$fileinfo
echo "<li><strong><a href=\"${header}\">" >>$fileinfo
echo "The processing header page</a></strong>" >>$fileinfo
echo "</ul>" >>$fileinfo
echo "<hr>" >>$fileinfo
#########################################################
# dump files once into temp files since dumping is slow
########################################################
rm -f info.tmp
$UTIL/dump_table $tapecat filename fileclas filesize >info.tmp
$UTIL/dump_table $tapecat filename descrip >descrip.tmp
################################
# loop over classes
################################
base=""
classes=$(awk '{print $2}' info.tmp |uniq )
for class in $classes; do
if [ -n "$base" ]; then
echo "</ul>" >>$fileinfo
echo "<hr>" >>$fileinfo
fi
echo "<h2> Files in the $class directory</h2>" >>$fileinfo
echo "<ul>" >>$fileinfo
################################
# loop over files of that class
################################
old=""
for file in $(awk 'word == $2 {print $1}' word=$class info.tmp ); do
base=${file%.wrap}
###################################
# determine class and description
###################################
class=$(awk '$1 == word {print $2}' word=$file info.tmp )
size=$(awk '$1 == word {print $3}' word=$file info.tmp )
descrip=$(awk '$1 == word ' word=$file descrip.tmp \
| sed -e "s/${file}//g" )
descrip=$(echo $descrip ) # removing whitespace
###################################
# is this a new file description?
##################################
if [ "$descrip" != "$old" ]; then
if [ -n "$old" ]; then
echo "</ol>" >>$fileinfo
fi
echo "<li><strong>${descrip}</strong>" >>$fileinfo
echo "<ol>" >>$fileinfo
old=$descrip
fi
###################################
# list the file
####################################
echo "<li><tt>${base}</tt>" >>$fileinfo
if [ $size -gt 0 ]; then
echo "(${size} Kilobytes)" >>$fileinfo
fi
done
echo "</ol>" >>$fileinfo
done
rm -f descrip.tmp
rm -f info.tmp
###############################
# HTML tail for fileinfo file
###############################
echo "</body>" >>$fileinfo
echo "</html>" >>$fileinfo
exit 0