filelists (version 2.1)
You can also look at:
#! /usr/bin/ksh
###########################################################################
#
# SYNTAX: filelists
#
# BRIEFLY: Make lists of the files to be distributed.
#
# DESCRIPTION: This routine lists the files.
# DESCRIPTION: First, it creates a FITS ASCII table list of all of the files
# DESCRIPTION: to be distributed. It then creates an HTML list of all of the
# DESCRIPTION: files, with information about each file.
# DESCRIPTION: Finally, it creates a FITS table listing all of the files
# DESCRIPTION: which will be saved as trend products.
#
# VERSION: 2.1
#
# HISTORY: 0.0 -> 1.0 8/19/96
# HISTORY: Removed the (.wrap) after wrapped file names
# HISTORY: in the HTML list (confusing for CD distributions).
# HISTORY: teldef now refers to both SIS and GIS files, so removed
# HISTORY: individual g2onflf, etc types.
# HISTORY:
# HISTORY: 1.0 -> 1.1 9/26/96
# HISTORY: Now reads list of file types to distribute from $LISTS/file_classes.
# HISTORY:
# HISTORY: 1.1 -> 1.2 11/26/96
# HISTORY: Added code to create the trendcat file. This code was moved from
# HISTORY: rev1trend.
# HISTORY:
# HISTORY: 1.2 -> 2.0 1/2/97
# HISTORY: Rewritten so that the tapecat is generated first, and then the
# HISTORY: HTML file list is created from it. The new HTML page requires that
# HISTORY: the file classes be sorted in $LISTS/file_classes.
# HISTORY:
# HISTORY: 2.0 -> 2.1 3/3/97
# HISTORY: Removed a problem, where files with the same description were
# HISTORY: getting listed separately in the HTML list, by removing
# HISTORY: whitespace around the description.
#
###########################################################################
#DEBUG=1
columns=columns.tmp
data=data.tmp
rm -f $data
rm -f $columns
$UTIL/milestone "Listing the files for distribution"
#########################################
# Save the job.par and process.par files
#########################################
jobpar=$( $UTIL/generate_filename jobpar )
parfile=$($UTIL/generate_filename parfile)
$UTIL/log_entry "Saving $JOBPAR as $jobpar and $PARFILE as $parfile"
cp $JOBPAR $jobpar
cp $PARFILE $parfile
##############################################################
# Create a dummy tape catalog file so that it can list itself
##############################################################
tapecat=$( $UTIL/generate_filename tapecat )
fileinfo=$($UTIL/generate_filename fileinfo)
header=$( $UTIL/generate_filename header )
rm -f $tapecat
rm -f $fileinfo
touch $tapecat
touch $fileinfo
#####################################
# Column info for tape catalog files
#####################################
rm -f $columns
echo "filenum I4" >> $columns
echo "filename A57" >> $columns
echo "filesize I7 kilobytes" >> $columns
echo "fileclas A20" >> $columns
echo "descrip A64" >> $columns
##############################
# List files to an ascii file
##############################
typeset -i nfiles=0
typeset -i count=0
typeset -i size=0
typeset -i total_size=0
typeset -i i=1
typelist=$( grep -v "^#" $LISTS/file_classes | \
awk -F'|' '$2 !~ /none / {print $1}')
for type in $typelist; do
description=$($UTIL/file_description $type)
class=$($UTIL/file_class $type)
###################
# Debugging output
###################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: type=$type
fi
###################################
# Loop over the files of each type
###################################
list=$($UTIL/any_filename $type)
if [ -n "$list" ]; then
count=$(echo $list | wc -w)
let nfiles=$nfiles+$count
for file in $list; do
###################
# Debugging output
###################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: file=$file
fi
#################################
# Get information about the file
#################################
format=$($STOOL/isFITS $file)
if [ "$type" = "tapecat" ]; then
format=FITS
fi
size=$(wc -c $file | awk '{print $1}')
let size=$size/1000
let total_size=$total_size+$size
######################
# Entry for FITS list
######################
if [ "$format" = "FITS" ]; then
echo "$i $file $size $class '$description' " >> $data
else
echo "$i ${file}.wrap $size $class '$description' " \
>> $data
fi
let i=$i+1
done
fi
done
let total_size=$total_size/1000
total_files=$nfiles
$UTIL/add_parameter $JOBPAR datasize "$total_size" i \
"Data set size in megabytes"
$UTIL/add_parameter $JOBPAR filecount "$total_files" i \
"Total number of distributed files"
###############################
# Create the tape catalog file
###############################
$UTIL/log_entry "Creating the FITS format file catalog $tapecat"
rm -f $tapecat
$UTIL/setup_parfile $FTOOL/fcreate.par cdfile=$columns \
datafile=$data \
outfile=$tapecat \
tbltype=ASCII
$FTOOL/fcreate >stdout.log 2>stderr.log
$UTIL/ftool_test fcreate $? $0 2 stdout.log stderr.log
rm -f $data
rm -f $columns
###########################################################################
###########################################################################
##
## Catalog the trend products
##
###########################################################################
###########################################################################
$UTIL/log_entry "Creating the trend product catalog"
columns=columns.tmp
data=data.tmp
rm -f $data
rm -f $columns
################################################################
# Create a dummy trend catalog file so that it can lists itself
################################################################
trendcat=$( $UTIL/generate_filename trendcat )
rm -f $trendcat
touch $trendcat
#####################################
# Column info for tape catalog files
#####################################
rm -f $columns
echo "filenum I4" >> $columns
echo "filename A57" >> $columns
echo "filesize I7 kilobytes" >> $columns
echo "fileclas A20" >> $columns
echo "descrip A64" >> $columns
##############################
# List files to an ascii file
##############################
typeset -i size=0
typeset -i i=1
typelist=$( grep -v "^#" $LISTS/file_classes | \
awk -F'|' '$3 !~ /none / {print $1}')
for type in $typelist; do
description=$($UTIL/file_description $type)
###################
# Debugging output
###################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: type=$type
fi
########################################################
# A file which goes to more than one directory has more
# than one class and is listed more than once
########################################################
for class in $($UTIL/trend_class $type); do
###################################
# Loop over the files of each type
###################################
list=$($UTIL/any_filename $type)
if [ -n "$list" ]; then
for file in $list; do
size=$(ls -s $file | awk '{print $1}')
echo "$i $file $size $class '$description' " >> $data
let i=$i+1
done
fi
done
done
################################
# Create the trend catalog file
################################
$UTIL/log_entry "Creating the FITS format file catalog $trendcat"
rm -f $trendcat
$UTIL/setup_parfile $FTOOL/fcreate.par cdfile=$columns \
datafile=$data \
outfile=$trendcat \
tbltype=ASCII
$FTOOL/fcreate >stdout.log 2>stderr.log
$UTIL/ftool_test fcreate $? $0 2 stdout.log stderr.log
rm -f $data
rm -f $columns
##############################################################################
##############################################################################
##
## Create the HTML fileinfo file from the tapecat
##
##############################################################################
##############################################################################
$UTIL/log_entry "Creating $fileinfo"
################################
# HTML header for fileinfo file
################################
seq=$($UTIL/read_parfile $JOBPAR sequence)
echo "<HTML>" >>$fileinfo
echo "<!-- This page generated by $0 on $(date) -->" >>$fileinfo
echo "<TITLE>" >>$fileinfo
echo "ASCA Sequence $seq File List" >>$fileinfo
echo "</TITLE>" >>$fileinfo
echo "<BODY>" >>$fileinfo
echo "<H1>ASCA Sequence $seq File List</H1>" >>$fileinfo
echo "This page lists all of the files included in the" >>$fileinfo
echo "distribution for this sequence." >>$fileinfo
echo "The size of most files is given after the" >>$fileinfo
echo "file name." >>$fileinfo
echo "In total there are $total_files files" >>$fileinfo
echo "occupying $total_size megabytes of storage space." >>$fileinfo
echo "<P>" >>$fileinfo
echo "The following information is also available:" >>$fileinfo
echo "<UL>" >>$fileinfo
echo " <LI><STRONG><A HREF=\"${header}\">" >>$fileinfo
echo " The processing header page</A></STRONG>" >>$fileinfo
echo "</UL>" >>$fileinfo
echo "<HR>" >>$fileinfo
########################################################
# Dump files once into temp files since dumping is slow
########################################################
rm -f info.tmp
$UTIL/dump_table $tapecat filename fileclas filesize >info.tmp
$UTIL/dump_table $tapecat filename descrip >descrip.tmp
####################
# Loop over classes
####################
base=""
classes=$(awk '{print $2}' info.tmp | uniq)
for class in $classes; do
if [ -n "$base" ]; then
echo "</UL>" >>$fileinfo
echo "<HR>" >>$fileinfo
fi
echo "<H2>Files in the $class directory</H2>" >>$fileinfo
echo "<UL>" >>$fileinfo
################################
# Loop over files of that class
################################
old=""
for file in $(awk 'word == $2 {print $1}' word=$class info.tmp); do
base=${file%.wrap}
####################
# debugging output
####################
if [ -n "$DEBUG" ]; then
echo ${0##*/}: file=$file
fi
##################################
# Determine class and description
##################################
class=$(awk '$1 == word {print $2}' word=$file info.tmp)
size=$(awk '$1 == word {print $3}' word=$file info.tmp)
descrip=$(awk '$1 == word ' word=$file descrip.tmp \
| sed -e "s/${file}//g")
descrip=$(echo $descrip) # removing whitespace
##################################
# Is this a new file description?
##################################
if [ "$descrip" != "$old" ]; then
if [ -n "$old" ]; then
echo " </OL>" >>$fileinfo
fi
echo " <LI><STRONG>${descrip}</STRONG>" >>$fileinfo
echo " <OL>" >>$fileinfo
old=$descrip
fi
################
# List the file
################
echo " <LI><TT>${base}</TT>" >>$fileinfo
if [ $size -gt 0 ]; then
echo " (${size} Kilobytes)" >>$fileinfo
fi
done
echo " </OL>" >>$fileinfo
done
rm -f descrip.tmp
rm -f info.tmp
##############################
# HTML tail for fileinfo file
##############################
echo "</BODY>" >>$fileinfo
echo "</HTML>" >>$fileinfo
exit 0