#!/usr/local/bin/perl
# Execute "perl pt.pl -h" for help
#
# Copyright: 2003-2006 by John R Larsen  -  john@larsen-family.us
# http://pingtest.sourceforge.net
# Released under the same Artistic license as perl

# $Id: pt.pl,v 4.29 2006/09/02 21:12:32 jlarsen Exp $
use strict;
use warnings;
use Errno qw(EAGAIN);
use Fcntl qw(:DEFAULT :flock);
use File::Copy;
use File::Compare;
use File::Basename;
use Getopt::Long qw(:config permute pass_through);
use POSIX ":sys_wait_h";

#------------------------------------------------------------------------------
# Define program wide variables here

# Constants defined here
use constant GOOD_PORT => 1;
use constant BAD_PORT => 0;
use constant GOOD => 1;
use constant BAD => 0;
use constant YES => 1;
use constant NO => 0;
use constant TRUE => 1;
use constant FALSE => 0;
use constant FOREVER => 1;


# Internal program wide variables declared here
my $script_ver;         # Initialized by get_version
my $script_date;        # Initialized by get_version
my $D = 0;              # Debug flags. Default is 0.
my $bad_port_cntr = 0;
my @db = ();            # database array
my $date;
my $debug_flags = 0;
my $dropped_cntr = 0;
my $dropped_email_sent = FALSE;
my $early_sleep_exit_cntr = 0;
my $email_address = 'none';
my $host;               # name of host ping_test is running on
my $IdentityFile = "/path/to/ping_test/id_rsa.private_key_name";
my $include_rrd_graphs = "";
my $ip_found = FALSE;
my $old_debug_flags;
my $pgm;
my $ping_rate = 60;
my $ping_time;
my $predicted_start_time;
my $rc;
my $rrdtool_graphs = FALSE;
my $test_ping_status = "ping_good";
my $threshold = 5;
my $time_of_ping;
my $toggle_cnt = 0;
my $traceroute_timeout = 60;
my $UserKnownHostsFile = "/path/to/ping_test/UserKnownHostsFile";
my $User = "user_login_name_on_webserver";
my $webpage_filename = "default";
my $webpage_local_dir = "none";
my $webpage_refresh_rate = "  ";
my $webpage_update_cntr = 0;
my $webpage_update_rate = 10;
my $webpage_remote_dir = "none";
my $webserver_domain_name = "webserver_WAN_name";
my $webserver_domain_port = 22;
my $while_start_epoch;
my $working_dir;


# Command line argument variables
my $debug;                     # Debug flags
my $ip = 'not_set';            # IP address to test


# Global variables that are set by values in pt.conf
# Global variables names are in uppercase
our $HOST_PGM = "not set";
our $LS_PGM = "not set";
our $MAIL_PGM = "not set";
our $NETSTAT_PGM = "not set";
our $PING_PGM = "not set";
our $PS_PGM = "not set";
our $SCP_PGM = "not set";
our $RRDTOOL_PGM = "not set";
our $SSH_PGM = "not set";
our $TELNET_PGM = "not set";
our $TRACE_ROUTE_PGM = "not set";
our $W_PGM = "not set";
our $ZIP_PGM = "not set";

# Constants used as indices into @db array.  See "&read_pt_db" for @db initialization.
# If the array changes, these constants will need to be changed as well as the formats
# in the subroutines &read_pt_db, &write_stats_file, and &check_week.
use constant TOTAL_PINGS => 0;
use constant GOOD_PINGS => 1;
use constant DROPPED_PINGS => 2;
use constant PERCENT_LOSS => 3;
use constant MIN_PING_TIME => 4;
use constant MIN_PING_DATE_YMD => 5;
use constant MIN_PING_DATE_HMS => 6;
use constant MIN_PING_DATE_DOW => 7;
use constant AVG_PING_TIME => 8;
use constant MAX_PING_TIME => 9;
use constant MAX_PING_DATE_YMD => 10;
use constant MAX_PING_DATE_HMS => 11;
use constant MAX_PING_DATE_DOW => 12;
use constant EMAILS_SENT => 13;
use constant LAST_EMAIL_YMD => 14;
use constant LAST_EMAIL_HMS => 15;
use constant LAST_EMAIL_DOW => 16;
use constant TEST_STARTED_YMD => 17;
use constant TEST_STARTED_HMS => 18;
use constant TEST_STARTED_DOW => 19;
use constant WEEK_NUMBER => 20;
use constant WEEK_START_DATE => 21;
use constant DAILY_STATS => 22;           # This points to the first variable of the daily stats
use constant WEEK_GOOD_IDX => 29;
use constant WEEK_DROPPED_IDX => 37;
use constant WEEK_LOSS_IDX => 45;
use constant WEEK_EMAILS_IDX => 53;
use constant WEEK_MIN_IDX => 61;
use constant WEEK_AVG_IDX => 69;
use constant WEEK_MAX_IDX => 77;
use constant SUMS_SUN => 78;
use constant SUMS_WEEK => 85;
use constant SUMS_OVERALL => 86;


#--- FUNCTIONS START HERE ARRANGED ALPHABETICALLY (except "main" is at end) ---

#------------------------------------------------------------------------------
# check_week:  Function that checks if in a new week
# 
sub check_week {
   my $func_name = "check_week";
   if ($D & 0x1) {printf ("[%d]$func_name:$$> Inside check_week\n", __LINE__);}

format WEEKS =
@<<<<<<<< - Good:    @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>>
$db[21],             $db[22],$db[23],$db[24],$db[25],$db[26],$db[27],$db[28],$db[29]
@<<<<<<<< - Dropped: @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>>
$db[21],             $db[30],$db[31],$db[32],$db[33],$db[34],$db[35],$db[36],$db[37]
@<<<<<<<< - Loss %:    @>>>>   @>>>>   @>>>>   @>>>>   @>>>>   @>>>>   @>>>>   @>>>>
$db[21],             $db[38],$db[39],$db[40],$db[41],$db[42],$db[43],$db[44],$db[45]
@<<<<<<<< - Emails:  @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>>
$db[21],             $db[46],$db[47],$db[48],$db[49],$db[50],$db[51],$db[52],$db[53]
@<<<<<<<< - Min-ms:  @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>>
$db[21],             $db[54],$db[55],$db[56],$db[57],$db[58],$db[59],$db[60],$db[61]
@<<<<<<<< - Avg-ms:  @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>>
$db[21],             $db[62],$db[63],$db[64],$db[65],$db[66],$db[67],$db[68],$db[69]
@<<<<<<<< - Max-ms:  @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>>
$db[21],             $db[70],$db[71],$db[72],$db[73],$db[74],$db[75],$db[76],$db[77]
---- @<<<< @<<<<<<<<< ------------- ping rate: @<< ----- threshold: @< ---------------
     $script_ver, $script_date,               $ping_rate,          $threshold
.

   # Get current week number and compare to value in @db.
   chomp (my $week = `date '+%U'`); 
   if ($week != $db[WEEK_NUMBER]) {
      # Getting here means the week numbers are different.

      # Update week number that appears in stats
      $db[WEEK_NUMBER] = $week;
      
      # Save current week's statistics at beginning of $ip.weeks
      open WEEKS, ">", "$ip.weeks.tmp" or die "Cannot open $ip.weeks.tmp: $!";
      write WEEKS;
      close WEEKS;
      if (-e "$ip.weeks") {
         system "cat $ip.weeks >> $ip.weeks.tmp";
      }
      rename "$ip.weeks.tmp", "$ip.weeks";

      # Initialize @db for a new week.  Zero out daily and week statistics.
      my $count = DAILY_STATS;
      while ($count <= WEEK_MAX_IDX) {
         $db[$count] = 0;
         $count++;
      }

      # Zero out daily and week sums
      $count = SUMS_SUN;
      while ($count <= SUMS_WEEK) {
         $db[$count] = 0;
         $count++;
      }
      
      # Update week date value that appears in the stats
      chomp ($week = `date '+%y\-%b\-%d'`);
      $db[WEEK_START_DATE] = $week;

      # Save database and statistics
      &write_pt_db;
      &write_stats_file;
   }
} # check_week


#------------------------------------------------------------------------------
# cron_processing:  This function is called each time pt.pl is invoked from the
# command line.  Normally this is done in a crontab.  If pt.pl isn't running
# for the IP on the command line then this function starts it back up.
sub cron_processing {
   my $func_name = "cron_processing";
   if ($D & 0x1) {printf ("[%d]$func_name:$$> Inside cron_processing\n", __LINE__);}
	my $pt_pid;

   # Add entry to logfile or create new if $ip.logfile doesn't exist
   my $time = &get_time;
	printf ("\n%s: cron processing\n\n", $time);
	&log_entry;

	# Get the PID of the running test
	$pt_pid = &get_pt_pid;

	# Initialize a $ip.heartbeat file if it doesn't exist
	if ( ! -e "$ip.heartbeat" ) {
      open IP_HEARTBEAT, ">", "$ip.heartbeat" or die "Cannot open file $ip.heartbeat: $!";
		print IP_HEARTBEAT "RUNNING";
      close IP_HEARTBEAT;
	}

   # Kill running process if $ip.heartbeat indicates the thread is stalled.  
   # The cron job writes NOT_RUNNING to $ip.heartbeat.  The main while loop
	# overwrites this with RUNNING.  The time between cron runs must be longer
	# than the ping rate for this to work.
   open IP_HEARTBEAT, "<", "$ip.heartbeat" or die "Cannot open file $ip.heartbeat: $!";
   chomp (my $heartbeat = <IP_HEARTBEAT>);
   close IP_HEARTBEAT;
	if ($heartbeat ne "RUNNING") {
		kill 9, $pt_pid;
	}

	# Check if pt test is still alive 
	$rc = &pid_active;
	if (! $rc) {

		# Send an email notifying that the test was down

      # Delete the tmp file if it already exists and open the file again
      unlink "/tmp/$$.email";
      open TMP_EMAIL, ">>", "/tmp/$$.email" or die "Cannot open file /tmp/$$.email: $!";

      # Build the message to be sent
		printf TMP_EMAIL ("pt: cron: No active pid. Restarting $host to $ip\n");
		my $status = `cat $ip.stats`;
      print TMP_EMAIL "$status";
		if ( -e "$ip.weeks" ) {
			$status = `cat $ip.weeks`;
         print TMP_EMAIL "$status";
		}

      # Do a traceroute to $ip and put it in the email
		&trace_route ($traceroute_timeout);
		if ( -e "$ip.route" ) {
         print TMP_EMAIL "----- traceroute ---------------------------------------------------------------\n";
			$status = `cat $ip.route`;
         print TMP_EMAIL "$status";
		}

      # Include the email log if it exists
		if (-e "$ip.emails") {
			print TMP_EMAIL "\n----- email log ---------------------------------------------------------------\n";
			$status = `cat $ip.emails`;
         print TMP_EMAIL "$status";
		}
      
      # All done making message so close the file
      close TMP_EMAIL;

      # Send the email if valid email address is in config file
      if ($email_address ne "none") {
         system "cat /tmp/$$.email | $MAIL_PGM -s \"pt: cron: $ip from $host - No active pid\" $email_address";
      }

      # Delete the temp file
      unlink "/tmp/$$.email";

		# Restart the test
      system "$working_dir/$ip -debug $D &";
		exit 0;
	} else {
		# Process still running. Update heartbeat and exit.
      open IP_HEARTBEAT, ">", "$ip.heartbeat" or die "Cannot open file $ip.heartbeat: $!";
		print IP_HEARTBEAT "NOT_RUNNING";
      close IP_HEARTBEAT;
		exit 0;
	}
} # cron_processing


#------------------------------------------------------------------------------
# display_debug_help:  Function to display debug help.
sub display_debug_help {
   my $func_name = "display_debug_help";
   if ($D & 0x1) {printf ("[%d]$func_name:$$> Inside display_debug_help\n", __LINE__);}

   # Pipe this help screen through more so it doesn't fly off the screen
   open MORE, "|more" or die "cannot pipe through more: $!";
   print MORE << "EOF";
----- Debug Help ($script_ver $script_date) ------------------------------------
Debug is enabled in one of three ways in descending order of precedence: 

   "-debug 0xNNNN" on command line
   PT_DEBUG_FLAGS environment variable
   "debug_flags" file in same directory as pt

Debug output can be changed "on the fly" by changing the contents of
"debug_flags", which is read each time pt goes through its while loop.

Debug output is added only to the logfile.  If you want to see in real time
what is going on then tail the logfile with this command: "tail -f
domain_name.logfile".

00000000 - No debug

0000_0001 - Enable all the "Inside function name" debug lines
0000_0002 - 
0000_0004 - Inside while loop message
0000_0008 - 

0000_0010 - main: Display environment each pass through while loop
0000_0020 - 
0000_0040 - 
0000_0080 - main: Toggle TEST_PING_STATUS every 5 pings for testing

0000_0100 - main: Output the contents of domain_name.stats
0000_0200 - main: Output the contents of domain_name.weeks
0000_0400 - main: Update and output the contents of domain_name.route
0000_0800 - 

0000_1000 - ping_ip: Output \$PING_TIME extracted from ping data
0000_2000 - sleep_time: Output runtime statistics
0000_4000 - pid_active: verbose ps output
0000_8000 - get_route: Enable debugging output

0001_0000 - reaper: Output pids of reaped zombies
0002_0000 - web_page: verbose output of scp webpage transfer
0004_0000 - web_page: disable certain webpage content

8000_0000 - Redirect all output to STDOUT instead of "logfile"


Copyright: 2003-2006 by John R Larsen  -  john\@larsen-family.us
http://pingtest.sourceforge.net
Released under the same Artistic license as perl

EOF
   close MORE;
   exit 0;
} # display_debug_help


#------------------------------------------------------------------------------
sub display_help {
my $func_name = "display_help";
if ($D & 0x1) {printf ("[%d]$func_name:$$> Inside display_help\n", __LINE__);}

# Pipe this help screen through more so it doesn't fly off the screen
open MORE, "|more" or die "cannot pipe through more: $!";
print MORE << "EOF";
----- ping_test ($script_ver $script_date) ------------------------------------
usage:
pt.pl <domain_name>  [ -debug n | -dl | -ds | -k | -mv IP | -s | -restart ]
pt.pl <-h | -hd | -hs | -ms n>

The domain_name is pinged at the ping rate defined in the 
domain_name.conf file.  Use the -hs option for help setting up. Read 
and edit the domain_name.conf file to configure the test as desired.

Options:
-debug n  debug level (ie. -d 0x4040 )
-dl       Delete Logs
-ds       Delete Statistics
-h        Display this Help screen
-hd       Help Debug.  This explains how debugging works.
-hs       Help Setup.  This walks you through how to set up a ping_test
          including remote webpage updates.
-k        Kill a running instance of pt.  The test will be restarted by
          cron unless you also remove the line from the crontab.
-ms n     Make summary webpages.  n is seconds between auto webpage refreshes.
          Enter 0 for n if auto refreshes aren't desired.  Copy pt.pl into 
          the "png" directory where the graphs are stored.  Webpages are 
          created in directory that group all available graphs of the same type 
          into one page.  The pages are created in the parent directory ".."
          above the png directory.
-mv IP    Rename all the files associated with this test to the IP entered.
          This is useful if the domain name or IP address changes and the
          files all need to be updated with the new name or IP.
-restart  Restart ping_test.  This kills the currently running ping_test and
          immediately restarts.  This is useful when changes have been made
          to the domain_name.conf file that need to be reloaded.
-s        Show statistics.  This outputs statistics in text format to STDOUT.

Copyright: 2003-2006 by John R Larsen  -  john\@larsen-family.us
http://pingtest.sourceforge.net
Released under the same Artistic license as perl

EOF
   close MORE;
   exit 0;
} # display_help


#------------------------------------------------------------------------------
# display_setup_help:  Function to display help on ssh_tunnel configuration.
sub display_setup_help {
   my $func_name = "display_setup_help";
   if ($D & 0x1) {printf ("[%d]$func_name:$$> Inside display_setup_help\n", __LINE__);}

   # Pipe this help screen through more so it doesn't fly off the screen
   open MORE, "|more" or warn "cannot pipe through more: $!";
   print MORE << "EOF";
----- Setup Help ($script_ver $script_date) -----------------------------------

Before ping_test will work there are a few configuration steps that must be
done.  Follow these steps on the machine where ping_test will run.  ping_test
has several different reporting methods: email, local webpage, and remote
webpage.  Each can be enabled separately.

1.  Create a directory and copy pt.pl into this directory.  A typical directory
name is "ping_test".  All generated files will be in this directory.  Files are
uniquely named so it is possible to have several instances of ping_test running
from the same directory.
    
2.  Edit the first line of pt.pl to point to where perl is installed on your
system.  The first time you run pt.pl from the command line it creates pt.conf,
which must be edited to specify program locations for your system.  A single
instance of pt.conf is used for all ping_test running in the same directory.
Error messages will be shown if programs can't be found.  Edit pt.conf with the
correct path information to the programs.

3.  The example internet address "domain.com" is used for the rest of this
setup help.  Next invoke pt.pl with the internet address that will be tested.  

         ./pt.pl domain.com

This creates the files shown below:

         domain.com -> pt.pl*
         domain.com.conf
         domain.com.logfile
         domain.com.pid
         domain.com.stats

Note that the symbolic link "domain.com" to "pt.pl" makes it possible to
simply use "domain.com" as the shell command.  This is a little easier than
always having to type "pt.pl domain.com".  The only place you must use
"pt.pl domain.com" is in a crontab line.

4.  Edit the "domain.com.conf" configuration file.  This allows you to specify
the email address where up/down notices are sent, the threshold number of
dropped pings in a row before an email is sent, the ping rate, whether or not
to generate local or remote webpages, whether or not to create graphics using
the round robin database tool (see RRDtool at http://oss.oetiker.ch/rrdtool),
and options used by scp and ssh to transfer files to a remote webserver if
desired.  Setting up for remote webserver updates is described at the end of
this setup help.
    
5.  The next step is to manually start up ping_test while tailing the logfile
to verify proper operation.  The logfile is the default location for all
program output.  In one shell use this command to tail the logfile:
    
       tail -f /path/to/ping_test/domain.com.logfile

In another shell use the following command to start up ping_test:

       /path/to/ping_test/domain.com
    
If all is working well you will see lots of information.  Look for any error
messages and fix the problems as needed.  You can set a debug flag that will
force ping_test to simulate dropped packets.  See the debug help page for
information (pt.pl -hd).

6.  Once manual operation is confirmed it is time to create a crontab entry so
the ping_test will be started automatically.  A typical crontab entry looks
like this:

     0,15,30,45 * * * * /path/to/ping_test/pt.pl domain.com
    
Note that "domain.com" is used as an argument to "pt.pl".  This is the correct
syntax for the crontab entry.  The above crontab line checks every 15 minutes
to see that ping_test is running and restarts it if the ping_test pid isn't
active.  When everything is working correctly the cron task simply exits.

7.  You can kill all processes associated with a ping_test with either of
these commands:

        /path/to/ping_test/pt.pl domain.com -k
        /path/to/ping_test/domain.com -k

8.  You can restart a ping_test with either of these commands:

        /path/to/ping_test/pt.pl domain.com -restart
        /path/to/ping_test/domain.com -restart

This is useful when you make changes to the domain.com.conf file and need to
restart ping_test right away.  This stops a current ping_test and restarts it
thereby reading the domain.com.conf file.


SETTING UP AND TESTING REMOTE WEBSERVER UPDATES

The last section in domain.com.conf sets up options needed by SSH and SCP to
securely copy files to a remote webserver.  You must have an account on the
webserver machine which is accessible using SSH with RSA public/private keys.
    
1.  Use the "ssh-keygen" utility to generate an RSA public/private key pair.
Name the key something like "id_rsa.machine_name" so it is easily identifiable.
Do not give the key a pass phrase.
    
2.  Copy public RSA key to your ~/.ssh/authorized_keys file on the remote
webserver machine.

3.  Put the private RSA key in the /path/to/ping_test directory.  Change to the
ping_test directory and set permissions to owner access only.  The RSA private
key file must only be readable by the owner of the file, ie. "chmod 400
filename".  The directory must only be writeable by the owner, ie. 700.  Use
chmod to set permissions accordingly.

4.  Modify the domain.com.conf file with the correct values for the following:
         WEBPAGE_REMOTE_DIR   /path/on/webserver/to/files
         HostName             webserver_WAN_name
         Port                 22
         UserKnownHostsFile   /path/to/ping_test/domain.com.known_hosts
         User                 user_login_name_on_webserver
         IdentityFile         /path/to/ping_test/id_rsa.private_key_name

After modifying the domain.com.conf file you must restart the ping_test so
that the domain.com.ssh file is recreated with the new information.
        
5.  Verify the ssh connection can be made.  If everything is setup correctly
you should be able to start a shell on the webserver without entering any
password.  In a shell use this command to connect to the webserver machine.
    
        ssh -F domain.com.ssh webpage
    
The above command verifies the settings in the domain.com.conf file.  You
will be warned that the ssh server's key doesn't exist and you will be
asked if you want to add it.  Answer yes.  Once this key is in the
known_hosts file you won't be asked for it anymore.  Type exit to return to
the ssh client shell.  
    
If errors occur, use the following command to make a verbose connection to
the webserver machine.  Carefully look through the debug output to see what
the problem is:
    
        ssh -v -F domain.com.ssh webpage

6.  Webpages and graphics will now automatically be copied over at the rate
specified by the WEBPAGE_UPDATE_RATE in domain.com.conf.  Graphics are stored
in WEBPAGE_REMOTE_DIR/png on the remote webserver and in WEBPAGE_LOCAL_DIR/png
locally.  These directories are made automatically if needed.
    

WEBPAGE SUMMARY PAGES

If many ping_tests are copying their webpages to the same webserver directory
it may be desireable to see all the pages at once.  There is an option "-ms n"
to pt.pl that creates summary webpages.  The way this works is to copy pt.pl 
into the WEBPAGE_REMOTE_DIR/png or the WEBPAGE_LOCAL_DIR/png directory.  In 
that directory invoke "pt.pl -ms n" where n is the number of seconds between 
auto webpage refreshes.  Enter 0 for n if auto refreshes are not desired.  
Invoking pt.pl in this way creates a set of summary webpages in the 
WEBPAGE_REMOTE_DIR and WEPAGE_LOCAL_DIR directories.  The summary pages have 
names like "summary_3hrs.html".  This webpage displays all the 
domain.com.3hrs.png graphs on one page.  This makes it easy to compare results
and is especially useful if domain.com is being tested from several different
locations.  "pt.pl -ms n" uses whatever png files are in the png directory at 
the time to create the summary pages.  If tests are added or deleted then the 
summary webpage files will need to be remade.


SUPPORTED PLATFORMS:

This script has been used successfully on the following operating systems:
    Solaris 7, 8, 9
    Linux 2.x on Mandrake 9.1, Yellow Dog 4.1, Gentoo
    Cygwin 1.5.x on Windows 2000 and Windows XP

Copyright: 2003-2006 by John R Larsen  -  john\@larsen-family.us
http://pingtest.sourceforge.net
Released under the same Artistic license as perl

EOF
   close MORE;
   exit 0;
} # display_setup_help


#------------------------------------------------------------------------------
# delete_logs:  Function that deletes all the log files
sub delete_logs {
   my $func_name = "delete_logs";
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside delete_logs\n", __LINE__) }
   # Only do this if IP address has been defined
   if ($ip ne '') {
      $rc = &yes_or_no ("Do you really want to delete logfiles");
      if ($rc == YES ) {
         print STDERR "Deleting logfiles\n";
         unlink "$ip.logfile";
         while (glob "$ip.logfile.*.zip") {
            unlink $_;
         }
      } else {
         print STDERR "Exiting without deleting logfiles\n";
      }
      exit 0;
   } else {
      warn "IP address must be first argument on command line\n";
      exit 1;
   }
} # delete_logs


#------------------------------------------------------------------------------
# delete_stats:  Function that deletes all the statistics files
sub delete_stats {
   my $func_name = "delete_stats";
   my $pt_pid;
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside delete_stats\n", __LINE__) }
   # Only do this if IP address has been defined
   if ($ip ne '') {
      # Make sure the process is killed before trying to rename the files
      $pt_pid = &get_pt_pid;
      $rc = &pid_active;
      if (! $rc) {
         $rc = &yes_or_no ("Do you really want to delete statistics");
         if ($rc == YES ) {
            print STDERR "Deleting statistics\n";
            unlink "$ip.db";
            unlink "$ip.emails";
            unlink "$ip.route";
            unlink "$ip.stats";
            unlink "$ip.weeks";
         } else {
            print STDERR "Exiting without deleting statistics\n";
         }
         exit 0;
      } else {
         print STDERR "ERROR: ping_test $pt_pid still running.  \nKill it first with \"$0 $ip -k\"\n";
         exit 1;
      }
   } else {
      warn "IP address must be first argument on command line\n";
      exit 1;
   }
} # delete_stats


#------------------------------------------------------------------------------
# display_env:  Function that displays current environment values
sub display_env {
   my $func_name = "display_env";
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside display_env\n", __LINE__) }
   printf ("----- CURRENT ENVIRONMENT VARIABLE VALUES -----------------\n");
   printf ("debug:                  0x%x\n", $D);
   printf ("email_address:          %s\n", $email_address);
   printf ("host:                   %s\n", $host);
   printf ("IdentityFile            %s\n", $IdentityFile);
   printf ("ip:                     %s\n", $ip);
   printf ("ping_rate:              %d\n", $ping_rate);
   printf ("rrdtool_graphs:         %d\n", $rrdtool_graphs);
   printf ("threshold:              %d\n", $threshold);
   printf ("traceroute_timeout:     %d\n", $traceroute_timeout);
   printf ("User:                   %s\n", $User);
   printf ("UserKnownHostsFile:     %s\n", $UserKnownHostsFile);
   printf ("webpage_filename:       %s\n", $webpage_filename);
   printf ("webpage_local_dir:      %s\n", $webpage_local_dir);
   printf ("webpage_refresh_rate:   %s\n", $webpage_refresh_rate);
   printf ("webpage_remote_dir:     %s\n", $webpage_remote_dir);
   printf ("webpage_update_rate:    %d\n", $webpage_update_rate);
   printf ("webserver_domain_name:  %s\n", $webserver_domain_name);
   printf ("webserver_domain_port:  %d\n", $webserver_domain_port);
   printf ("working_dir:            %s\n", $working_dir);

} # display_env


#------------------------------------------------------------------------------
# display_stats:  Function that displays stats associated with a given ping_test
sub display_stats {
   my $func_name = "display_stats";
   my $status;
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside display_stats\n", __LINE__) }
   # Only do this if IP address has been defined
   if ($ip ne '') {
      $status = `cat $ip.stats`;
      print STDERR "$status";
      if ( -e "$ip.weeks" ) {
         $status = `cat $ip.weeks`;
         print STDERR "$status";
      }
      if ( -e "$ip.emails" ) {
         print STDERR "\n----- email log ---------------------------------------------------------------\n";
         $status = `cat $ip.emails`;
         print STDERR "$status";
      }
      exit 0;
   } else {
      warn "IP address must be first argument on command line\n";
      exit 1;
   }
} # display_stats


#------------------------------------------------------------------------------
# get_pt_pid: Function that returns the pid for "pt" extracted from $ip.pid
sub get_pt_pid {
   my $func_name = "get_pt_pid";
   my $pt_pid;
   if ( -s "$ip.pid" ) {
      open IP_PID, "<", "$ip.pid" or die "Cannot open file $ip.pid: $!";
      chomp ($pt_pid = <IP_PID>);
      close IP_PID;
   } else {
      # Return an impossible pid if file doesn't exist
      $pt_pid = 66666;
   }
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside get_pt_pid (pid: $pt_pid)\n", __LINE__) }
   return $pt_pid;
} # get_pt_pid


#-------------------------------------------------------------------------------
# get_version.  Function that parses script version and date info from the RCS Id line.
sub get_version {
   my $Id;
   # The cvs version string is embedded in $version_info below
   my $version_info = '$Id: pt.pl,v 4.29 2006/09/02 21:12:32 jlarsen Exp $';
    
   # Extract the version and date from the string
   $version_info =~ m{v ([\d\.]+) (\d+/\d+/\d+)};
   #                  ^^                         Match the letter v followed by a space "v "
   #                     ^^^^^^^                 Match any number of digits and periods
   #                             ^               Match a single space
   #                               ^^^^^^^^^^^   Match any number of digits followed by a slash,
   #                                             followed by any number of digits and a slash,
   #                                             followed by any number of digits.  YYYY/MM/DD
   $script_ver = "v".$1;  # Stick the letter "v" to the front of the the version number
   $script_date = $2;
} # get_version


#-------------------------------------------------------------------------------
# get_time.  Function that returns date and time
sub get_time {
   chomp (my $time = `date '+%Y\-%b\-%d\ %H\:%M\:%S\ %a'`);
   return $time;
} # get_time


#-------------------------------------------------------------------------------
# install.  Function that installs pt.pl and walks user through all the setup steps
sub install {
   print "debug123> inside install\n";
   print "debug124> $ENV{perl_pgm}\n";
   open TEMP, ">", "./temp.$$" or die "Cannot open ./temp$$ for writing: $!";
   print TEMP "#!$ENV{perl_pgm}\n";
   close TEMP;
   system "cat ./pt.pl >> temp.$$";
   system "chmod 755 temp.$$";
   system "mv temp.$$ pt.pl";
   exit 0;
} # install


#------------------------------------------------------------------------------
# init_debug:  Initialize debug.  Debug flags can be set on the command line,
# through and environment variable, or through the file debug_flags in that
# order of precedence.  The default destination for debug is into the file
# "logfile".  If bit 0x8000_0000 is set then all output is sent to STDOUT
# instead.
sub init_debug {
   my $func_name="init_debug";

   # Read value from file debug_flags and store it away
   if (-s "debug_flags") {
      open (DEBUG_FLAGS, "<", "debug_flags") 
         or die "Can't open debug_flags file: $!";

      # Read the value from the file, convert to a hex number and save for later use
      $old_debug_flags = <DEBUG_FLAGS>;
      close DEBUG_FLAGS;
      chomp $old_debug_flags;
      $old_debug_flags = oct $old_debug_flags;
      $debug_flags = $old_debug_flags;
      $D = $debug_flags;
   } else {
      $old_debug_flags = "0";
      $debug_flags = "0";
      $D = 0;
   }

   # Check if PT_DEBUG_FLAGS env variable exists and use it if it does
   if (exists $ENV{PT_DEBUG_FLAGS}) {
      $D = oct $ENV{PT_DEBUG_FLAGS};
   }

   # Setup where log output goes
   &set_log_dest;

} # init_debug


#------------------------------------------------------------------------------
# kill_ip:  Function that kills the ping_test for an IP
sub kill_ip {
   my $func_name = "kill_ip";
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside kill_ip\n", __LINE__) }
   # Only do this if IP address has been defined
   if ($ip ne '') {
      my $pt_pid = &get_pt_pid;
      $rc = &yes_or_no ("Do you really want to kill pid $pt_pid");
      if ($rc == YES ) {
         $rc = kill 9, $pt_pid;
         if ($rc) {
            print "Killed pid $pt_pid using -k from command line\n";
            print STDERR "Killed pid $pt_pid using -k\n";
         } else {
            print STDERR "No such process: $pt_pid\n";
         }
      } else {
         print STDERR "Exiting without killing pid\n";
      }
      exit 0;
   } else {
      warn "IP address must be first argument on command line\n";
      exit 1;
   }
} # kill_ip


#------------------------------------------------------------------------------
# make_ip_conf: Function to create a template $IP.conf file
sub make_ip_conf {
   my $func_name = "make_ip_conf";
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside make_ip_conf\n", __LINE__) }
   my $webpage_filename = $ip . "_from_" . $host;
   rename ("$ip.conf", "$ip.conf.save") if (-e "$ip.conf");
   open IP_CONF, ">", "$ip.conf" or die "Cannot open $ip.conf for writing: $!";
   print IP_CONF << "EOF";
# NOT CONFIGURED - DELETE THIS LINE ONCE THIS FILE HAS BEEN CONFIGURED
# Filename:  $ip.conf
#
# Description:  This configuration file provides information for this instance 
# of pt.pl.  There is a separate IP.conf file for each IP address being tested.
# It is important not to change the formatting of the file because it is 
# processed using perl which is expecting things to be in certain locations.
#
# Copyright: 2003-2006 by John R Larsen  -  john\@larsen-family.us
# http://pingtest.sourceforge.net
# Released under the same Artistic license as perl
#
##############################################################################
# The following section contains configuration information.
#
# The email addresses below receive diagnostic messages.  Separate
# multiple addresses with commas and no white space.  The word "none"
# turns off email sending and is the default.
#
EMAIL_ADDRESS     none


# The threshold defined below is how many dropped pings in a row are required
# before an email is sent.  The default is 5.
#
DROPPED_THRESHOLD    5


# The ping rate in seconds can be changed here.  The default is 60 seconds.
#
PING_RATE    60


# The trace route timeout value in seconds can be changed here.  The default is 
# 60 seconds.  Window's tracert is very slow for some reason.
#
TRACEROUTE_TIMEOUT    60


##############################################################################
# WEBPAGE UPDATES
# Webpages can be created and copied to a local directory on the machine
# where ping_test is running and/or to a directory on a remote webserver.
# A value of "none" turns off the webpage copy.  If both are "none" then
# webpage creation is disabled.  Put a valid directory in place of "none" 
# to enable this feature.  Note that for remote webpage updates to work
# the SSH OPTIONS section at the end of this file must have valid values.
# Execute "pt.pl -hs" for detailed setup information.
#
WEBPAGE_REMOTE_DIR  none
WEBPAGE_LOCAL_DIR   none


# The filename of the webpage copied to the webserver defaults to
# $webpage_filename.  To choose a different name replace the
# word "default" on the line below with the desired name.  Do NOT 
# include the extension ".html" because the name is used in other
# places.  The extension will be appended automatically.
#
WEBPAGE_FILENAME  default


# The webpage update rate is in number of pings.  This is how often all 
# the webpages are copied over to the remote webserver or copied into the
# local directory.  The default is 10 pings which works out to every 
# ten minutes with the default ping rate of 60 seconds.  Adjust the value 
# as desired.
#
WEBPAGE_UPDATE_RATE   10


# The webpage refresh rate is the value in seconds that is put in the META 
# tag that forces the webpage to auto refresh.  Put "none" in as a value if 
# auto refresh isn't desired.  The default is 600 seconds.
#
WEBPAGE_REFRESH_RATE   600


# Daily, weekly, monthly, and yearly graphs in png format can be made if 
# RRDtool (http://oss.oetiker.ch/rrdtool) is available on the system.   
# The default is "false".  Set RRDTOOL_GRAPHS to "true" and graphs will be 
# created.  The graphs are copied into WEBPAGE_REMOTE_DIR/png directory and
# into WEBPAGE_LOCAL_DIR/png directory.  The size of archives in the RRD 
# database file is a function of the PING_RATE.  If the PING_RATE is changed 
# then the $ip.rrd file must be deleted so that a new
# one is created.  Be aware that all previous RRD history is lost when the 
# $ip.rrd file is deleted.
#
RRDTOOL_GRAPHS false


##############################################################################
# SSH OPTIONS
# Below are the ssh options needed for remote webserver updates.  Refer to the
# setup help page (pt.pl -hs) for instructions on how to setup and test this.
# The information in this section is used to created $ip.ssh
# which is used by SSH and SCP to make secure unassisted connections.

# Replace "webserver_WAN_name" with the internet name of your webserver
HostName webserver_WAN_name

# The normal port number for ssh and scp is 22.  Change the value here 
# if needed.
Port 22

# Change the path below to where the $ip.known_hosts 
# file is located
UserKnownHostsFile /path/to/ping_test/$ip.known_hosts

# Change the User to the name used to log into the webserver
User user_login_name_on_webserver

# Change the path below to where the private key file is located
IdentityFile /path/to/ping_test/id_rsa.private_key_name
EOF
   close IP_CONF;
   print STDERR "Created $ip.conf     - You must edit this file.\n";
} # make_ip_conf


#------------------------------------------------------------------------------
# make_ip_ssh: Function to create $ip.ssh file.  It creates a temp version and
# only replaces the existing file if the contents have changed.  This keeps 
# the timestamps from changing.
sub make_ip_ssh {
   my $func_name = "make_ip_ssh";
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside make_ip_ssh\n", __LINE__) }
   open IP_SSH, ">", "$ip.ssh.temp" or die "Cannot open $ip.ssh.temp for writing: $!";
   print IP_SSH << "EOF";
# Filename:  $ip.ssh
# DO NOT EDIT THIS FILE.  It is auto generated by pt.pl from the contents of
# $ip.conf.  Edit that file instead if changes need to be made.
#
# Description:  This file is used by SSH and SCP to make secure noninteractive
# connections to a remote webserver.  Execute "pt.pl -hs" for setup help.
#
# Copyright: 2003-2006 by John R Larsen  -  john\@larsen-family.us
# http://pingtest.sourceforge.net
# Released under the same Artistic license as perl
#
##############################################################################
Host webpage
HostName = $webserver_domain_name
Port = $webserver_domain_port
UserKnownHostsFile = $UserKnownHostsFile
User = $User
IdentityFile = $IdentityFile
EOF
   close IP_SSH;

   if (-e "$ip.ssh") {
      # Check if the newly created $ip.ssh.temp file is different than the existing file
      if (compare ("$ip.ssh.temp", "$ip.ssh")) {
         # Getting here means the files were different and $ip.ssh needs to be updated
         move "$ip.ssh.temp", "$ip.ssh" or die "Moving $ip.ssh.temp to $ip.ssh failed: $!";
      }
      else {
         # The files are the same so get rid of the temp file
         unlink "$ip.ssh.temp";
      }
   }
   else {
      # The file doesn't exist so mv the temp version to be used
      move "$ip.ssh.temp", "$ip.ssh" or die "Moving $ip.ssh.temp to $ip.ssh failed: $!";
   }
} # make_ip_ssh


#------------------------------------------------------------------------------
# log_entry: Function that adds an entry to the logfile
sub log_entry {
   my $status;
	my $func_name = "log_entry";
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside log_entry\n", __LINE__) }
	$status = `cat $ip.stats`;
   print "$status";
   if ( -e "$ip.weeks" ) {
		$status = `cat $ip.weeks`;
      print "$status";
   }
   if ( -e "$ip.emails" ) {
      print "\n----- email log ---------------------------------------------------------------\n";
      system "cat $ip.emails >> $ip.logfile";
   }
} # log_entry


#------------------------------------------------------------------------------
# logfile:  Function that compresses and rotates logs weekly
sub logfile {
   my $func_name = "logfile";
   my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks);
   my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst);
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside logfile\n", __LINE__) }

   # Compress logfile once a week and save 4 compressed logs
   my $current_time = time();
   ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime ($current_time);

   # Only rotate logs Sunday morning between midnight and 1:00 AM
   if (($wday == 0) && ($hour == 0) && (-e "$ip.logfile")) {
      # It's in the right hour for checking.  Only rotate logs if $ip.logfile.1.zip exists.
      if (-e "$ip.logfile.1.zip") {
         # Get the time info for the file.  We're interested in $mtime, the modified date.
         ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks)
            = stat ("$ip.logfile.1.zip");

         # Make sure the old $ip.logfile.1.zip is at least 60 * 60 * 164 seconds old.  This is two
         # hours short of a full week.  That will allow for DST.  This will only happen once because
         # the new logfile will have a time stamp very close to the current time.
         if (($current_time - $mtime) > (60 * 60 * 164)) {

            # Rotate the old logfiles
            my $cnt_2 = 4;  # Number of compressed logfiles to keep around
            my $cnt_1 = $cnt_2 - 1;
            while ($cnt_1 > 0) {
               if (-e "$ip.logfile.$cnt_1.zip") {
                  rename "$ip.logfile.$cnt_1.zip", "$ip.logfile.$cnt_2.zip"
               }
               $cnt_1--;
               $cnt_2--;
            }

            if (-e "$ip.logfile") {
               # Append current stats and email log to the logfile
               system "cat $ip.stats >> $ip.logfile";
               if (-e "$ip.weeks") {
                  system "cat $ip.weeks >> $ip.logfile";
               }
               if (-e "$ip.emails") {
                  system "cat $ip.emails >> $ip.logfile";
               }
               # Close logfile before compressing
               close LOG;

               # Compress the logfile
               system "$ZIP_PGM $ip.logfile.1.zip $ip.logfile";

               # Reopen the logfile and set size to zero
               &set_log_dest ("clear");
            }

            # Send email with weekly stats
            &send_weekly_email;
         }
      } else {
         # No $ip.logfile.1.zip so compress logfile and create new one
         if (-e "$ip.logfile") {
            # Append current stats and email log to the logfile
            system "cat $ip.stats >> $ip.logfile";
            if (-e "$ip.weeks") {
               system "cat $ip.weeks >> $ip.logfile";
            }
            if (-e "$ip.emails") {
               system "cat $ip.emails >> $ip.logfile";
            }
            # Close logfile before compressing
            close LOG;

            # Compress the logfile
            system "$ZIP_PGM $ip.logfile.1.zip $ip.logfile";

            # Reopen the logfile and set size to zero
            &set_log_dest ("clear");
         }

         # Send email with weekly stats
         &send_weekly_email;
      }
   }
} # logfile


#------------------------------------------------------------------------------
# make_summary_pages: Function that creates the summary webpages that display all
# the RRDtool graphs grouped on one page per type.  pt.pl must be copied into
# the WEBPAGE_REMOTE_DIR/png or the WEBPAGE_LOCAL_DIR/png directory and executed 
# from there.
sub make_summary_pages {
   sub make_page {
      my $summary_type = $_[0];
      my $summary_refresh_rate = $_[1];
      open HTML, ">", "../summary_$summary_type.html" or die "Cannot open summary_$summary_type.html for writing: $!";
      print HTML << "      EOF";
      <HTML>
      <HEAD>
      <TITLE>Summary of $summary_type ping_test graphs</TITLE>
      $summary_refresh_rate
      </HEAD>
      <BODY>
      <h1> Summary of $summary_type ping_test graphs</h1>
      <table>
      EOF

      my $first = 0;

      while (glob "*.$summary_type.png") {
         $_ =~ s/\.$summary_type\.png//;
         if ($first == 0) {
            $first = 1;
            print HTML << "            EOF";
            <tr>
               <td>
                  <A HREF="png/$_.$summary_type.large.png"><IMG SRC="png/$_.$summary_type.png" title="Click for larger graph"></A>
               </td>
            EOF
         }
         else {
            $first = 0;
            print HTML << "            EOF";
               <td>
                  <A HREF="png/$_.$summary_type.large.png"><IMG SRC="png/$_.$summary_type.png" title="Click for larger graph"></A>
               </td>
            </tr>
            EOF
         }
      }

      if ($first == 0) {
         print HTML << "         EOF";
            </table>
            </BODY>
            </HTML>
         EOF
      }
      else {
         print HTML << "         EOF";
               </tr>
            </table>
            </BODY>
            </HTML>
         EOF
      }
      close HTML;
   } # make_page

   my $refresh_rate = $_[0];
   my $summary_refresh_rate = "   ";
   if ($refresh_rate != 0) {
      $summary_refresh_rate = "<META HTTP-EQUIV=\"refresh\" content=\"$refresh_rate\">";
   }

   &make_page ("3hrs", "$summary_refresh_rate");
   &make_page ("3hrs.loss", "$summary_refresh_rate");
   &make_page ("24hrs", "$summary_refresh_rate");
   &make_page ("24hrs.loss", "$summary_refresh_rate");
   &make_page ("week", "$summary_refresh_rate");
   &make_page ("week.loss", "$summary_refresh_rate");
   &make_page ("4week", "$summary_refresh_rate");
   &make_page ("4week.loss", "$summary_refresh_rate");
   &make_page ("year", "$summary_refresh_rate");
   &make_page ("year.loss", "$summary_refresh_rate");

   exit 0;
} # make_summary_pages


#------------------------------------------------------------------------------
# pid_active: Function that uses pid and path from $ip.pid and tests
# if the process is still active.  
# Returns: pid value if active or 0 inactive
sub pid_active {
   my $func_name = "pid_active";
   my $pid;
   my $path;
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside pid_active\n", __LINE__) }
   open IP_PID, "<", "$ip.pid" or warn "Cannot open file $ip.pid for reading: $!";
   chomp ($pid = <IP_PID>);
   chomp ($path = <IP_PID>);
   close IP_PID;

   # Use PS_PGM to test if the process is still alive
   my $ps = `$PS_PGM -edf`;
   if ($D & 0x4000) { printf ("[%d]$func_name:$$> Testing for pid $pid.  Output of ps command:\n$ps\n", __LINE__) }
   if ($ps =~ /$pid.*$path/) {
      return $pid;
   } 
   else {
      return 0;
   }
} # pid_active


#------------------------------------------------------------------------------
# pid_save: Function that saves pid $$ and program $0 to file $ip.pid
sub pid_save {
   my $func_name = "pid_save";
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside pid_save (pid: $$  path: $0)\n", __LINE__) }
   open IP_PID, ">", "$ip.pid" or die "Cannot open file $ip.pid for writing: $!";
   print IP_PID "$$\n$working_dir/$ip\n";
   close IP_PID;
} # pid_save


#------------------------------------------------------------------------------
# ping_ip: Function that pings the $ip.  It returns ping time if the ping was 
# successful or returns -1 if the ping failed.  The ping time and ping result 
# are output to wherever logging is going.
sub ping_ip {
   my $func_name = "ping_ip";
   my $ping_time;
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside ping_ip\n", __LINE__) }

	# Capture the time the ping is performed
   $time_of_ping = &get_time;

	# ping the $ip using $PING_PGM which has correct syntax for OS running on
   my $ping_result = `$PING_PGM`;

   # Replace newlines with spaces to make logfile easier to read
   $ping_result =~ s/\n/ /g;

   # Save ping time and ping results to the logfile and to STDOUT if debug enabled
   print "$time_of_ping - $ping_result\n";

   # Isolate the ping time. The ping command has different output based on OS type.
   # The ping times are in the form of 123/123/123 which is min/avg/max ping times.
   # Search for this data pattern.
   $rc = $ping_result =~ /(\d+\.?\d*\/)/;
   chop ($ping_time = $1);

   # Display extracted ping time if debugging is turned on
   if ($D & 0x1000) { printf ("[%d]$func_name:$$> \$ping_time: $ping_time\n", __LINE__) }

   # Check if the regex matched.
   if ($rc) {
      return $ping_time;
   } else {
      return -1;
   }
} # ping_ip


#------------------------------------------------------------------------------
# process_cmd_line:  Function that uses GetOptions to process the command line
sub process_cmd_line {
# Process a command line full of arguments
   my $func_name = "process_cmd_line";
   # Reconfigure Getopt for no pass through so that any command line errors 
   # get caught.
   Getopt::Long::config ("no_pass_through");
   my $restart = FALSE;
   $rc = GetOptions ( 
      'debug=o' => sub {$D = $_[1]; &set_log_dest},
      'ds' => sub {&delete_stats},
      'dl' => sub {&delete_logs},
      'k' => sub {&kill_ip},
      'mv=s' => sub {&rename_files},
      's' => sub {&display_stats},
      'restart' => sub {$restart = TRUE},
      '<>' => \&process_arg
   );
    
   # Die if GetOptions reports an error
   if (! $rc) {
      print STDERR "ERROR: Incorrect command line.  Unable to continue.\n";
      exit 1;
   }

   # Non option arguments processed here.  
   sub process_arg {
      if ( $ip_found == FALSE ) {
         $ip_found = TRUE;
         $ip = $_[0];
      } else {
         die "More than one non option argument on command line\n";
      }
   }

   # Check if -restart was on command line.  Need to do it here to be sure that the IP
   # is found before invoking the function.
   if ($restart == TRUE) {
      &restart;
      exit 0;
   }
} # process_cmd_line


#------------------------------------------------------------------------------
# read_ip_conf: Function to read settings from $ip.conf file
sub read_ip_conf {
   my $func_name = "read_ip_conf";
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside read_ip_conf\n", __LINE__) }
   if ( -e "$ip.conf" ) {
      open IP_CONF, "<", "$ip.conf" or die "Cannot open $ip.conf for reading: $!";
      while (<IP_CONF>) {
         if (/^# NOT CONFIGURED/) {
            print STDERR "You must edit $ip.conf! The first line still says NOT CONFIGURED\n";
            exit 1;
         }

         if (/^EMAIL_ADDRESS\s*(\S*)/) {
            $email_address = $1;
            next;
         }

         if (/^DROPPED_THRESHOLD\s*(\S*)/) {
            $threshold = $1;
            next;
         }

         if (/^PING_RATE\s*(\S*)/) {
            $ping_rate = $1;
            next;
         }

         if (/^TRACEROUTE_TIMEOUT\s*(\S*)/) {
            $traceroute_timeout = $1;
            next;
         }

         if (/^WEBPAGE_REMOTE_DIR\s*(\S*)/) {
            $webpage_remote_dir = $1;
            next;
         }

         if (/^WEBPAGE_FILENAME\s*(\S*)/) {
            $webpage_filename = $1;
            next;
         }

         if (/^WEBPAGE_LOCAL_DIR\s*(\S*)/) {
            $webpage_local_dir = $1;
            next;
         }

         if (/^WEBPAGE_UPDATE_RATE\s*(\S*)/) {
            $webpage_update_rate = $1;
            $webpage_update_cntr = $webpage_update_rate;
            next;
         }

         if (/^WEBPAGE_REFRESH_RATE\s*(\S*)/) {
            if ($1 eq "none") {
              $webpage_refresh_rate = "  ";
            }
            else {
              $webpage_refresh_rate = "<META HTTP-EQUIV=\"refresh\" content=\"$1\">";
            }
            next;
         }

         if (/^RRDTOOL_GRAPHS\s*(\S*)/) {
            if ($1 =~ m/TRUE/i) {
               $rrdtool_graphs = TRUE;
            }
            else {
               $rrdtool_graphs = FALSE;
            }
            next;
         }

         # The following lines get the information needed to create the $ip.ssh file.
         # This will grab the last "HostName" value in the file.  This
         # is the webserver host information.
         if (/^HostName\s*(\S*)/) {
            $webserver_domain_name = $1;
         }

         # This will grab the last "Port" value in the file.  This
         # is the webserver host information.
         if (/^Port\s*(\S*)/) {
            $webserver_domain_port = $1;
         }

         # This will grab the last "UserKnownHostsFile" value in the file.
         if (/^UserKnownHostsFile\s*(\S*)/) {
            $UserKnownHostsFile = $1;
         }

         # This will grab the last "User" value in the file.
         if (/^User\s*(\S*)/) {
            $User = $1;
         }

         # This will grab the last "IdentityFile" value in the file.
         if (/^IdentityFile\s*(\S*)/) {
            $IdentityFile = $1;
         }
      }
      close IP_CONF;

      # Check if webpage_filename is still "default".  If so, change it.
      if ($webpage_filename eq "default") {
         $webpage_filename = $ip . "_from_" . $host;
      }

      # Create $ip.ssh.temp file.
      &make_ip_ssh;
   }
   else {
      # Create $ip.conf since it doesn't exist
      &make_ip_conf;

      # Check for necessary symbolic link and create it if missing
      eval { symlink ("",""); 1 } or die "Can't make symbolic links: $!";
      symlink ("pt.pl", "$ip") if (! -e "$ip");

      # Save the $ip.pid file so that it will exist
      &pid_save;

      # Touch the $ip.logfile so that it will exist
      system "touch $ip.logfile";

      # Touch the $ip.stats file so that it will exist
      system "touch $ip.stats";

      # Remove the extraneous "not_set.logfile" so it doesn't clutter things up
      unlink "not_set.logfile";

      # Give a short help message the first time through pointing the user what to do next
      print STDERR << "EOF";

Welcome to ping_test.  You are reading this because you ran pt.pl for an
IP without its required configuration file.  It has been created, but
there is still some setup that needs to be done.  Please use the "-hs" 
option for setup help.  The "-h" option gives general help.

EOF
      exit 0;
   }
} # read_ip_conf


#------------------------------------------------------------------------------
# read_pt_db.  Function that reads the $IP.db file or creates one if id doesn't exist.
sub read_pt_db {
   my $func_name = "read_pt_db";
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside read_pt_db\n", __LINE__) }


   if ( ! -e "$ip.db" ) {
      # If the statistics file doesn't exist then initialize variables with default values.
      # Using push would be easier, but need to keep track of what is in which index value.
      $db[0] = 0;                             #TOTAL_PINGS
      $db[1] = 0;                             #GOOD_PINGS
      $db[2] = 0;                             #DROPPED_PINGS
      $db[3] = 0;                             #PERCENT_LOSS
      $db[4] = '10000.0';                     #MIN_PING_TIME
      $db[5] = 'YYYY-MMM-DD';                 #MIN_PING_DATE_YMD
      $db[6] = 'HH:MM:SS';                    #MIN_PING_DATE_HMS
      $db[7] = 'DDD';                         #MIN_PING_DATE_DOW
      $db[8] = 0;                             #AVG_PING_TIME
      $db[9] = 0;                             #MAX_PING_TIME
      $db[10] = 'YYYY-MMM-DD';                #MAX_PING_DATE_YMD
      $db[11] = 'HH:MM:SS';                   #MAX_PING_DATE_HMS
      $db[12] = 'DDD';                        #MAX_PING_DATE_DOW
      $db[13] = 0;                            #EMAILS_SENT
      $db[14] = 'YYYY-MMM-DD';                #LAST_EMAIL_YMD
      $db[15] = 'HH:MM:SS';                   #LAST_EMAIL_HMS
      $db[16] = 'DDD';                        #LAST_EMAIL_DOW
      # Put test start date in $db[17]        #TEST_STARTED_YMD
      #                        $db[18]        #TEST_STARTED_HMS
      #                        $db[19]        #TEST_STARTED_DOW
      my $time = &get_time;
      push @db, split ' ', $time;

      chomp ($time = `date '+%U'`); 
      $db[20] = $time;                        #WEEK_NUMBER
      chomp ($time = `date '+%y\-%b\-%d'`);
      $db[21] = $time;                        #WEEK_START_DATE
      # The week's data starts at $db[22]     #DAILY_STATS
      #             S M T W T F S Week
      push @db, qw/ 0 0 0 0 0 0 0 0 /;        #Good counts
      push @db, qw/ 0 0 0 0 0 0 0 0 /;        #Bad counts
      push @db, qw/ 0 0 0 0 0 0 0 0 /;        #Percent loss
      push @db, qw/ 0 0 0 0 0 0 0 0 /;        #Number of emails sent
      push @db, qw/ 0 0 0 0 0 0 0 0 /;        #min ping time in ms
      push @db, qw/ 0 0 0 0 0 0 0 0 /;        #avg ping time in ms
      push @db, qw/ 0 0 0 0 0 0 0 0 /;        #max ping time in ms

      # Initialize the sums entries that hold the ping time sums for average calculations
      $db[78] = 0;                            # Day of Week 0 - Sunday     #SUMS_SUN
      $db[79] = 0;                            # Day of Week 1 - Monday
      $db[80] = 0;                            # Day of Week 2 - Tuesday
      $db[81] = 0;                            # Day of Week 3 - Wednesday
      $db[82] = 0;                            # Day of Week 4 - Thursday
      $db[83] = 0;                            # Day of Week 5 - Friday
      $db[84] = 0;                            # Day of Week 6 - Saturday
      $db[85] = 0;                            # Week sum                   #SUMS_WEEK
      $db[86] = 0;                            # Overall sum                #SUMS_OVERALL


      # Save the newly initialized values to disk.
      open IP_DB, ">", "$ip.db" or die "Cannot open file $ip.db: $!";
      print IP_DB "@db";
      close IP_DB;
      # Initialize the $ip.stats file since a new $ip.db just got created
      &write_stats_file;
   } else {
      open IP_DB, "<", "$ip.db" or die "Cannot open file $ip.db: $!";
      @db = split / /, <IP_DB>;
      close IP_DB;
   }
} # read_pt_db


#------------------------------------------------------------------------------
# reaper: Function that reaps zombie processes
# Requires:  use POSIX ":sys_wait_h";
sub reaper {
	my $func_name = "reaper";
   my $zombie;
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside reaper\n", __LINE__) }
   while (FOREVER) {
      $zombie = waitpid (-1, WNOHANG);
      last if ($zombie == -1);
      last if ($zombie == 0);
      if ( $D & 0x1_0000 ) { printf ("[%d]$func_name:$$> $pgm: Reaped zombie: $zombie\n", __LINE__) }
   }
} # reaper


#------------------------------------------------------------------------------
# rename_files:  Function that renames files associated with a test
sub rename_files {
   my $func_name = "rename_files";
   my $pt_pid;
   if (! $_[1]) {
      print STDERR "ERROR: Must provide new name.\n";
      exit 1;
   }
   my $new_name = $_[1];
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside rename_files: $new_name\n", __LINE__) }

   # Only do this if IP address has been defined
   if ($ip ne '') {
      # Make sure the process is killed before trying to rename the files
      $rc = &pid_active;
      if (! $rc) {
         $rc = &yes_or_no ("Do you really want to rename $ip.filename to $new_name.filename");
         if ($rc == YES ) {
            print STDERR "Renaming files...\n";
            if ( -e "$ip")               { rename "$ip",               "$new_name" }
            if ( -e "$ip.conf")          { rename "$ip.conf",          "$new_name.conf" }
            if ( -e "$ip.db")            { rename "$ip.db",            "$new_name.db" }
            if ( -e "$ip.emails")        { rename "$ip.emails",        "$new_name.emails" }
            if ( -e "$ip.heartbeat")     { rename "$ip.heartbeat",     "$new_name.heartbeat" }
            if ( -e "$ip.html")          { rename "$ip.html",          "$new_name.html" }
            if ( -e "$ip.known_hosts")   { rename "$ip.known_hosts",   "$new_name.known_hosts" }
            if ( -e "$ip.logfile")       { rename "$ip.logfile",       "$new_name.logfile" }
            if ( -e "$ip.logfile.1.zip") { rename "$ip.logfile.1.zip", "$new_name.logfile.1.zip" }
            if ( -e "$ip.logfile.2.zip") { rename "$ip.logfile.2.zip", "$new_name.logfile.2.zip" }
            if ( -e "$ip.logfile.3.zip") { rename "$ip.logfile.3.zip", "$new_name.logfile.3.zip" }
            if ( -e "$ip.logfile.4.zip") { rename "$ip.logfile.4.zip", "$new_name.logfile.4.zip" }
            if ( -e "$ip.pid")           { rename "$ip.pid",           "$new_name.pid" }
            if ( -e "$ip.route")         { rename "$ip.route",         "$new_name.route" }
            if ( -e "$ip.rrd")           { rename "$ip.rrd",           "$new_name.rrd" }
            if ( -e "$ip.rrd.html")      { rename "$ip.rrd.html",      "$new_name.rrd.html" }
            if ( -e "$ip.stats")         { rename "$ip.stats",         "$new_name.stats" }
            if ( -e "$ip.weeks")         { rename "$ip.weeks",         "$new_name.weeks" }
            print STDERR "Files renamed to $new_name.filename\n";
            print STDERR "You must manually edit $new_name.conf to change instances of $ip to $new_name\n";
         } else {
            print STDERR "Exiting without renaming files\n";
         }
         exit 0;
      } else {
         print STDERR "ERROR: ping_test $pt_pid still running.  \nKill it first with \"$0 $ip -k\"\n";
         exit 1;
      }
   } else {
      warn "IP address must be first argument on command line\n";
      exit 1;
   }
} # rename_files


#------------------------------------------------------------------------------
# restart: Function invoked when -restart is on command line.  Useful way to
# force ping_test to restart so that an updated pt.pl and/or $ip.conf file is used.
sub restart {
	my $func_name = "restart";
   my $pt_pid = &get_pt_pid;
   $rc = kill 9, $pt_pid;
   if ($rc) {
      printf ("[%d]$func_name:$$> Killed pid $pt_pid using -restart from command line\n", __LINE__,);
      print STDERR "Stopping $ip ($pt_pid)\n";
   } else {
      print STDERR "$ip ($pt_pid) not running\n";
   }

   # Restart the test
   printf ("[%d]$func_name:$$> Restarting $ip using -restart from command line\n", __LINE__,);
   print STDERR "Starting $ip\n";
   system "$working_dir/$ip -debug $D &";
   exit 0;
} # restart


#------------------------------------------------------------------------------
# rrd_create: Function that creates the RRD if rrdtools exist
sub rrd_create {
	my $func_name = "rrd_create";
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside rrd_create\n", __LINE__) }
   if (($rrdtool_graphs == TRUE) && (-x $RRDTOOL_PGM)) {
      if (! -e "$working_dir/$ip.rrd") {
         my $num_daily_rrd_samples = (60 * 60 * 24 * 7 / $ping_rate) + 5;
         my $num_weekly_rrd_samples = (60 * 60 * 24 * 7 * 4 / $ping_rate) + 5;
         my $num_yearly_rrd_samples = (60 * 60 * 24 / $ping_rate) + 5;
         my $unknown_time = (2 * $ping_rate) - 5;
         my $command = 
            "$RRDTOOL_PGM create $ip.rrd " .
            "-s $ping_rate " .
            "DS:ping_time:GAUGE:$unknown_time:0:10000 " .
            "DS:daily_loss:GAUGE:$unknown_time:0:100 " .
            "DS:weekly_loss:GAUGE:$unknown_time:0:100 " .
            "DS:yearly_loss:GAUGE:$unknown_time:0:100 " .
            # Archive for daily resolution. Store 7 days worth with CF of 1 sample.
            "RRA:MAX:0.5:1:$num_daily_rrd_samples " .
            # Archive for weekly resolution. Store 4 weeks worth with CF of 7 samples.
            "RRA:MAX:0.5:7:$num_weekly_rrd_samples " .
            # Archive for yearly resolution. Store 52 weeks worth with CF of 52 samples.
            "RRA:MAX:0.5:52:$num_yearly_rrd_samples " .
            "   ";
         system "$command";
      }
   }
} # rrd_create


#------------------------------------------------------------------------------
# rrd_graph: Function that generates the graphs if rrdtools exist
sub rrd_graph {
	my $func_name = "rrd_graph";
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside rrd_graph\n", __LINE__) }
   if (($rrdtool_graphs == TRUE) && (-x $RRDTOOL_PGM)) {
      # Make sure database exists
      &rrd_create;

      # Initialize the variable for the main webpage so the link shows up
      $include_rrd_graphs = "<h2><A HREF=\"png/$webpage_filename.html\">Click Here for Graphs</A></h2>";

      # Get the time these graphs are all made
      chomp (my $graph_time = `date '+%G %b %e %H%M'`); 

      # Create a small ping time graph for the last 3 hours
      my $command = 
         "$RRDTOOL_PGM graph $webpage_filename.3hrs.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-s -3hour " .
         "-t 'Ping Times For Last 3 Hours' " .
         "-v 'msec' " .
         "DEF:ping_time=$ip.rrd:ping_time:MAX " .
         "LINE1:ping_time#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a large ping time graph for the last 3 hours
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.3hrs.large.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-h 300 -w 800 " .
         "-s -3hour " .
         "-t 'Ping Times For Last 3 Hours' " .
         "-v 'msec' " .
         "DEF:ping_time=$ip.rrd:ping_time:MAX " .
         "LINE1:ping_time#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a small loss graph for the last 3 hours
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.3hrs.loss.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-s -3hour " .
         "-t 'Percent Packet Loss For Last 3 Hours' " .
         "-v 'percent' " .
         "DEF:daily_loss=$ip.rrd:daily_loss:MAX " .
         "LINE1:daily_loss#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a large loss graph for the last 3 hours
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.3hrs.loss.large.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-h 300 -w 800 " .
         "-s -3hour " .
         "-t 'Percent Packet Loss For Last 3 Hours' " .
         "-v 'percent' " .
         "DEF:daily_loss=$ip.rrd:daily_loss:MAX " .
         "LINE1:daily_loss#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a small ping time graph for the last 24 hours
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.24hrs.png " .
         " -i " .
         " -l 0 " .
         "-a PNG " .
         "-s -24hour " .
         "-t 'Ping Times For Last 24 Hours' " .
         "-v 'msec' " .
         "DEF:ping_time=$ip.rrd:ping_time:MAX " .
         "LINE1:ping_time#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a large ping time graph for the last 24 hours
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.24hrs.large.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-h 300 -w 800 " .
         "-s -24hour " .
         "-t 'Ping Times For Last 24 Hours' " .
         "-v 'msec' " .
         "-h 300 -w 800 " .
         "DEF:ping_time=$ip.rrd:ping_time:MAX " .
         "LINE1:ping_time#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a small loss graph for the last 24 hours
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.24hrs.loss.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-s -24hour " .
         "-t 'Percent Packet Loss For Last 24 Hours' " .
         "-v 'percent' " .
         "DEF:daily_loss=$ip.rrd:daily_loss:MAX " .
         "LINE1:daily_loss#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a large loss graph for the last 24 hours
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.24hrs.loss.large.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-h 300 -w 800 " .
         "-s -24hour " .
         "-t 'Percent Packet Loss For Last 24 Hours' " .
         "-v 'percent' " .
         "DEF:daily_loss=$ip.rrd:daily_loss:MAX " .
         "LINE1:daily_loss#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a small ping time graph for the last 1 week
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.week.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-s -1week " .
         "-t 'Maximum Ping Times For The Last Week' " .
         "-v 'msec' " .
         "DEF:ping_time=$ip.rrd:ping_time:MAX " .
         "LINE1:ping_time#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a large ping time graph for the last 1 week
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.week.large.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-s -1week " .
         "-t 'Maximum Ping Times For The Last Week' " .
         "-v 'msec' " .
         "-h 300 -w 800 " .
         "DEF:ping_time=$ip.rrd:ping_time:MAX " .
         "LINE1:ping_time#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a small loss graph for the last week
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.week.loss.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-s -1week " .
         "-t 'Percent Packet Loss For The Last Week' " .
         "-v 'percent' " .
         "DEF:weekly_loss=$ip.rrd:weekly_loss:MAX " .
         "LINE1:weekly_loss#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a large loss graph for the last week
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.week.loss.large.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-h 300 -w 800 " .
         "-s -1week " .
         "-t 'Percent Packet Loss For The Last Week' " .
         "-v 'percent' " .
         "DEF:weekly_loss=$ip.rrd:weekly_loss:MAX " .
         "LINE1:weekly_loss#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a small ping time graph for the last 4 weeks
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.4week.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-s -4week " .
         "-t 'Maximum Ping Times For The Last Four Weeks' " .
         "-v 'msec' " .
         "DEF:ping_time=$ip.rrd:ping_time:MAX " .
         "LINE1:ping_time#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a large ping time graph for the last 4 weeks
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.4week.large.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-s -4week " .
         "-t 'Maximum Ping Times For The Last Four Weeks' " .
         "-v 'msec' " .
         "-h 300 -w 800 " .
         "DEF:ping_time=$ip.rrd:ping_time:MAX " .
         "LINE1:ping_time#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a small loss graph for the last 4 weeks
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.4week.loss.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-s -4week " .
         "-t 'Percent Packet Loss For The Last Four Weeks' " .
         "-v 'percent' " .
         "DEF:weekly_loss=$ip.rrd:weekly_loss:MAX " .
         "LINE1:weekly_loss#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a large loss graph for the last 4 weeks
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.4week.loss.large.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-h 300 -w 800 " .
         "-s -4week " .
         "-t 'Percent Packet Loss For The Last Four Weeks' " .
         "-v 'percent' " .
         "DEF:weekly_loss=$ip.rrd:weekly_loss:MAX " .
         "LINE1:weekly_loss#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a small ping time graph for the last year
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.year.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-s -1year " .
         "-t 'Maximum Ping Times For The Last Year' " .
         "-v 'msec' " .
         "DEF:ping_time=$ip.rrd:ping_time:MAX " .
         "LINE1:ping_time#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a large ping time graph for the last year
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.year.large.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-s -1year " .
         "-t 'Maximum Ping Times For The Last Year' " .
         "-v 'msec' " .
         "-h 300 -w 800 " .
         "DEF:ping_time=$ip.rrd:ping_time:MAX " .
         "LINE1:ping_time#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a small loss graph for the last year
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.year.loss.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-s -1year " .
         "-t 'Percent Packet Loss For The Last Year' " .
         "-v 'percent' " .
         "DEF:yearly_loss=$ip.rrd:yearly_loss:MAX " .
         "LINE1:yearly_loss#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Create a large loss graph for the last year
      $command = 
         "$RRDTOOL_PGM graph $webpage_filename.year.loss.large.png " .
         "-i " .
         "-l 0 " .
         "-a PNG " .
         "-h 300 -w 800 " .
         "-s -1year " .
         "-t 'Percent Packet Loss For The Last Year' " .
         "-v 'percent' " .
         "DEF:yearly_loss=$ip.rrd:yearly_loss:MAX " .
         "LINE1:yearly_loss#FF0000 " .
         "COMMENT:\"  $graph_time  -  $ip from $host\" " .
         "2>&1 1>/dev/null";
      system "$command";

      # Make the webpage for rrd graphs
      my $date = &get_time;
      open IP_RRD, ">", "$ip.rrd.html" or die "Cannot open $ip.rrd.html for writing: $!";
      print IP_RRD << "EOF";
      <HTML>
      <HEAD>
         <TITLE>Graphical Ping Statistics:  $host to $ip</TITLE>
         $webpage_refresh_rate
      </HEAD>
      <BODY>
         <table width="1000">
            <tr>
               <td>
                  <center>
                     <h1> $host to $ip</h1>
                     <h6> ping_test $script_ver $script_date</h6>
                  </center>
               </td>
            </tr>
            <tr>
               <td>
                  <center>
                     <h2> Time: $date</h2>
                  </center>
               </td>
            </tr>
         </table>

         <table>
            <tr>
               <td>
                  <A HREF="$webpage_filename.3hrs.loss.large.png"><IMG SRC="$webpage_filename.3hrs.loss.png" title="Click for larger graph"></A>
               </td>
               <td>
                  <A HREF="$webpage_filename.3hrs.large.png"><IMG SRC="$webpage_filename.3hrs.png" title="Click for larger graph"></A>
               </td>
            </tr>
            <tr>
               <td>
                  <A HREF="$webpage_filename.24hrs.loss.large.png"><IMG SRC="$webpage_filename.24hrs.loss.png" title="Click for larger graph"></A>
               </td>
               <td>
                  <A HREF="$webpage_filename.24hrs.large.png"><IMG SRC="$webpage_filename.24hrs.png" title="Click for larger graph"></A>
               </td>
            </tr>
            <tr>
               <td>
                  <A HREF="$webpage_filename.week.loss.large.png"><IMG SRC="$webpage_filename.week.loss.png" title="Click for larger graph"></A>
               </td>
               <td>
                  <A HREF="$webpage_filename.week.large.png"><IMG SRC="$webpage_filename.week.png" title="Click for larger graph"></A>
               </td>
            </tr>
            <tr>
               <td>
                  <A HREF="$webpage_filename.4week.loss.large.png"><IMG SRC="$webpage_filename.4week.loss.png" title="Click for larger graph"></A>
               </td>
               <td>
                  <A HREF="$webpage_filename.4week.large.png"><IMG SRC="$webpage_filename.4week.png" title="Click for larger graph"></A>
               </td>
            </tr>
            <tr>
               <td>
                  <A HREF="$webpage_filename.year.loss.large.png"><IMG SRC="$webpage_filename.year.loss.png" title="Click for larger graph"></A>
               </td>
               <td>
                  <A HREF="$webpage_filename.year.large.png"><IMG SRC="$webpage_filename.year.png" title="Click for larger graph"></A>
               </td>
            </tr>
         </table>
      </BODY> </HTML>
EOF
      close IP_RRD;

      # Make local copies if $webpage_local_dir is not "none"
      if ($webpage_local_dir ne "none") {

         # Make sure the "png" directory exists by trying to creat it
         mkdir "$webpage_local_dir/png";

         # Copy the graphics webpage over
         copy "$ip.rrd.html", "$webpage_local_dir/png/$webpage_filename.html";

         # Copy all the graphics over
         while (glob "$ip*png") {
            copy "$_", "$webpage_local_dir/png/$_";
         }
      }

      # Only do the webserver update if $webpage_remote_dir isn't "none"
      return 0 if ($webpage_remote_dir eq "none");
      
      # Only attempt an scp copy if the port checks out good
      my $port_state = &test_port ($webserver_domain_name, $webserver_domain_port);
      if ($port_state != GOOD_PORT) {
         printf ("[%d]$func_name:$$> Webserver $webserver_domain_name port $webserver_domain_port is bad\n", __LINE__);
         return 1;
      }

      my $verbose = ($D & 0x2_0000) ? "-v" : "";
      my $redirection = ($D & 0x2_0000) ? "" : "2>/dev/null";
      
      # Make sure the "png" directory exists by creating it
      $rc = system "$SSH_PGM -F $working_dir/$ip.ssh -n webpage \"mkdir $webpage_remote_dir/png\" 2>/dev/null";

      # Copy the graphics webpage over
      $rc = system "$SCP_PGM -p $verbose -F $working_dir/$ip.ssh $ip.rrd.html webpage:$webpage_remote_dir/png/$webpage_filename.html $redirection";

      # Copy all the graphics over to the webserver
      while (glob "$ip*png") {
         $rc |= system "$SCP_PGM -p $verbose -F $working_dir/$ip.ssh $_ webpage:$webpage_remote_dir/png/$_ $redirection";
      }
      # Log success or failure
      if ($rc) {
         printf ("[%d]$func_name:$$> ERROR: RRD image copy failed to $webserver_domain_name\n", __LINE__);
         return 1;
      }
      else {
         printf ("[%d]$func_name:$$> RRD images copied to $webserver_domain_name\n", __LINE__);
         return 0;
      }
   }
} # rrd_graph


#------------------------------------------------------------------------------
# rrd_update: Function that updates the graphs if rrdtools exist
sub rrd_update {
   my $ping_time = $_[0];
   my $daily_loss_percent = $_[1];
   my $weekly_loss_percent = $_[2];
   my $yearly_loss_percent = $_[3];
	my $func_name = "rrd_update";
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside rrd_update\n", __LINE__) }

   if (($rrdtool_graphs == TRUE) && (-x $RRDTOOL_PGM)) {
      # Make sure database exists
      &rrd_create;

      # Calculate the time for the rrd update.  This is based on the number of seconds since
      # the epoch and needs to be modulo the step time so that no interpolating is done by rrd.
      my $current_time = time;
      my $update_time = $current_time - ($current_time % $ping_rate);

      # Update RRD database with results from this ping
      if ($ping_time ge 0) {
         my $command = 
            "$RRDTOOL_PGM update $ip.rrd " .
            "$update_time:" .
            "$ping_time:" .
            "$daily_loss_percent:" .
            "$weekly_loss_percent:" .
            "$yearly_loss_percent";
         system "$command";
      }
      else {
         # A negative $ping_time means the ping was dropped.  Need to put an Unknown value in for ping time.
         my $command = 
            "$RRDTOOL_PGM update $ip.rrd " .
            "$update_time:" .
            "U:" .
            "$daily_loss_percent:" .
            "$weekly_loss_percent:" .
            "$yearly_loss_percent";
         system "$command";
      }
   }
} # rrd_update



#------------------------------------------------------------------------------
# send_email: Function that sends the up or down email
# Arg1 - The email subject, ie. Down or Up
sub send_email {
	my $func_name = "send_email";
   my $up_down = $_[0];
   my $status;
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside send_email: $_[0]\n", __LINE__) }

   # Capture the time that this email is being sent
   my $email_time = &get_time;

   # Put a message in the logfile
	printf ("%s - $up_down: $host to $ip.  Email sent to $email_address.  Threshold: $threshold   Ping rate: $ping_rate\n", $email_time);
	$status = `cat $ip.stats`;
   print "$status";
	if (-e "$ip.weeks") {
		$status = `cat $ip.weeks`;
      print "$status";
	}
    
   # Do a traceroute to $ip and put it in the email
   &trace_route ($traceroute_timeout);
   if ( -e "$ip.route" ) {
      print "----- traceroute ---------------------------------------------------------------\n";
      $status = `cat $ip.route`;
      print "$status";
   }

   # Add an up or down line to the $ip.emails file
   open EMAILS, ">>", "$ip.emails" or die "Cannot open file $ip.emails: $!";
	printf EMAILS ("%s - $up_down: $host to $ip.  Email sent to $email_address.  Threshold: $threshold   Ping rate: $ping_rate\n", $email_time);
	if ($up_down eq "Down") {
		print EMAILS "---------------------------------------- $script_ver $script_date ----- \n"
	}
   close EMAILS;
   
   # Increment the emails counter and update the database and stats files
	$db[EMAILS_SENT]++;
   &write_stats_file;
   &write_pt_db;

   # Open a temp file for creating the email message.  Delete any old temp file that might be around.
   unlink "/tmp/$$.email";
   open TMP_EMAIL, ">>", "/tmp/$$.email" or die "Cannot open file /tmp/$$.email: $!";

	# Create the text of the email that will be sent
   # At the top of the email we want two lines giving the Up and Down times
	print TMP_EMAIL "$email_time - $up_down\n";
	if ($up_down eq "Down") {
		print TMP_EMAIL "$db[LAST_EMAIL_YMD] $db[LAST_EMAIL_HMS] $db[LAST_EMAIL_DOW] - Up\n";
	} else {
		print TMP_EMAIL "$db[LAST_EMAIL_YMD] $db[LAST_EMAIL_HMS] $db[LAST_EMAIL_DOW] - Down\n";
	}

   # Add the statistics file to the email
	$status = `cat $ip.stats`;
   print TMP_EMAIL "$status";
	if (-e "$ip.weeks") {
		$status = `cat $ip.weeks`;
      print TMP_EMAIL "$status";
	}
   
   # Do a traceroute to $ip and put it in the email
   &trace_route ($traceroute_timeout);
   if ( -e "$ip.route" ) {
      print TMP_EMAIL "----- traceroute ---------------------------------------------------------------\n";
      $status = `cat $ip.route`;
      print TMP_EMAIL "$status";
   }

   # Add the email log to the email
	print TMP_EMAIL "\n----- email log ---------------------------------------------------------------\n";
	if (-e "$ip.emails") {
		$status = `cat $ip.emails`;
      print TMP_EMAIL "$status";
	}
   close TMP_EMAIL;
   
   # Send the email if email is enabled in $ip.conf
   if ($email_address ne "none") {
      system "cat /tmp/$$.email | $MAIL_PGM -s \"pt: $up_down: $ip from $host\" $email_address";
   }
    # Remove the temp file
    unlink "/tmp/$$.email";

   # Record time email was sent in the database
   push my @time_array,  split / /, $email_time;
   $db[LAST_EMAIL_YMD] = $time_array[0];
   $db[LAST_EMAIL_HMS] = $time_array[1];
   $db[LAST_EMAIL_DOW] = $time_array[2];

   # Update the stats and database files
	&write_stats_file;
   &write_pt_db;
} # send_email


#------------------------------------------------------------------------------
# send_weekly_email: Function that sends the weekly summary email
sub send_weekly_email {
	my $func_name = "send_weekly_email";
   my $up_down = $_[0];
   my $status;
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside send_weekly_email\n", __LINE__) }

   return if $email_address eq "none";

   open TMP_EMAIL, ">", "/tmp/$$.email" or die "Cannot open file /tmp/$$.email: $!";
   my $time_of_email = &get_time;
   print TMP_EMAIL "$time_of_email - Weekly summary\n";
   $status = `cat $ip.stats`;
   print TMP_EMAIL "$status";
   if ( -e "$ip.weeks" ) {
      $status = `cat $ip.weeks`;
      print TMP_EMAIL "$status";
   }
   if ( -e "$ip.emails" ) {
      print TMP_EMAIL "----- email log ---------------------------------------------------------------\n";
      $status = `cat $ip.emails`;
      print TMP_EMAIL "$status";
   }
   close TMP_EMAIL;

   # Send the email and remove the temp file
   system "cat /tmp/$$.email | $MAIL_PGM -s \"pt: Weekly stats for $ip from $host\" $email_address";
   unlink "/tmp/$$.email";
} # send_weekly_email


#------------------------------------------------------------------------------
# set_debug_output:  Function that checks contents of file debug_flags and
# updates $D if the value has changed.
sub set_debug_output {
   my $func_name="set_debug_output";

   # Check the existence of a file named "debug_flags" and use it if conditions are right
   if (-s "debug_flags") {
      open (DEBUG_FLAGS, "<", "debug_flags") 
         or die "Can't open debug_flags file: $!";

      # Read the value from the file, convert to a hex number and save for later use
      chomp ($debug_flags = <DEBUG_FLAGS>);
      close DEBUG_FLAGS;
      $debug_flags = oct $debug_flags;
   } else {
      $debug_flags="0";
   }

   # Make changes if contents of file debug_flags has changed since the last reading
   if ($debug_flags != $old_debug_flags) {
      $D = $debug_flags;
      # If MSB of debug_flags has changed then need to change log destination
      if (($old_debug_flags & 0x80000000) != ($debug_flags & 0x80000000)) {
         &set_log_dest;
      }
      # Save current flags value for later use
      $old_debug_flags = $debug_flags;
      # Report flags changed if the IP address isn't null
      if ($ip ne "") {
         my $date = &get_time;
         printf ("\n[%d]$func_name:$$> set_debug_level: Debug flags changed to: 0x%x at %s\n\n", __LINE__, $D, $date);
      }
   }

   # This line is at the end because $D is set in this function
   if (($ip ne "") && (($D & 0x1) != 0)) {
      printf ("[%d]$func_name:$$> Inside set_debug_output (Flags: 0x%x)\n", __LINE__, $D);
   }
} # set_debug_output


#------------------------------------------------------------------------------
# set_log_dest:  Function that sets log output destination based on D$
# Arg1 - "clear" clears logfile else the logfile is opened for append
sub set_log_dest {
   my $clear = $_[0] ? $_[0] : "append";
   # The MSB of D$ determines where program output goes.  If the bit is clear (default)
   # then output goes to "logfile".  If it is set then output goes to STDOUT.
   if ($D & 0x8000_0000) {
      # Getting here means the bit is set and output goes to STDOUT
      close LOG;
      open LOG, ">-"
         or die "Can't redirect LOG output to STDOUT: $!";
      select LOG;
      $| = 1; #Flush log entries immediately
      select STDOUT;
   } else {
      # Getting here means the bit is clear and output goes to the logfile
      close LOG;
      if ($clear eq "clear") {
         # open the logfile with ">" to zero it out
         open LOG, ">", "$ip.logfile" or warn "Can't open logfile: $!";
         close LOG;
      }
      # Open filehandle to logfile and set autoflush on for logfile
      open LOG, ">>", "$ip.logfile" or warn "Can't open logfile for append: $!";
      select LOG;
      $| = 1; #Flush log entries immediately
   }
} # set_log_dest


#------------------------------------------------------------------------------
# set_working_dir: Function that sets the WORKING_DIR based on $0
sub set_working_dir {
   # Configure working directory
   $working_dir=`dirname $0`;
   chomp $working_dir;
   if ($working_dir eq ".") {
	   $working_dir=`pwd`;
      chomp $working_dir;
   }
   chdir $working_dir;

   # Configure the program name
   chomp ($pgm=basename ($0));

   # If $pgm isn't "pt.pl" then it's the domain name or IP address 
   # so set $ip to be that value
   if ( $pgm ne "pt.pl" ) {
      $ip = $pgm;
   }
   
} # set_working_dir


#-------------------------------------------------------------------------------
# setup_env.  Function that reads in a configuration file that sets up where
# all the host resident programs are located and various filter variables
# used by different OS.
sub setup_env {
my $func_name="setup_env";
if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside setup_env\n", __LINE__) }

my $file = "pt.conf";
my $pgm_filter = '^(\S*)';    # Regex to strip off any trailing arguments to the program
my $program;                  # Holds the program name less any arguments to the program
my $return;

# Check for existence of configuration file.  If it doesn't exist then create one.
if ( -s $file ) {
   # Process configuration file
   unless ($return = do $file) {
          die "couldn't parse $file: $@" if $@;
          die "You must edit $file to select your operating system"    unless defined $return;
          die "couldn't run $file"       unless $return;
   }

   # Verify programs read in from configuration file are valid and executable or die
   # Verify $HOST_PGM
   $HOST_PGM =~ /not set/ and die "Check $file.  Value for \$HOST_PGM needs to be set\n";
   $HOST_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or die "Check $file.  Value for \$HOST_PGM: \"$program\" not executable: $!\n";
   # Get the name of the host and remove the newline
   chomp ($host = `$HOST_PGM`);

   # Verify $LS_PGM
   $LS_PGM =~ /not set/ and die "Check $file.  Value for \$LS_PGM needs to be set\n";
   $LS_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or die "Check $file.  Value for \$LS_PGM: \"$program\" not executable: $!\n";

   # Verify $MAIL_PGM
   $MAIL_PGM =~ /not set/ and die "Check $file.  Value for \$MAIL_PGM needs to be set\n";
   $MAIL_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or die "Check $file.  Value for \$MAIL_PGM: \"$program\" not executable: $!\n";

   # Verify $NETSTAT_PGM
   $NETSTAT_PGM =~ /not set/ and die "Check $file.  Value for \$NETSTAT_PGM needs to be set\n";
   $NETSTAT_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or die "Check $file.  Value for \$NETSTAT_PGM: \"$program\" not executable: $!\n";

   # Verify $PING_PGM
   $PING_PGM =~ /not set/ and die "Check $file.  Value for \$PING_PGM needs to be set\n";
   $PING_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or die "Check $file.  Value for \$PING_PGM: \"$program\" not executable: $!\n";

   # Verify $PS_PGM
   $PS_PGM =~ /not set/ and die "Check $file.  Value for \$PS_PGM needs to be set\n";
   $PS_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or die "Check $file.  Value for \$PS_PGM: \"$program\" not executable: $!\n";

   # Verify $RRDTOOL_PGM
   $RRDTOOL_PGM =~ /not set/ and die "Check $file.  Value for \$RRDTOOL_PGM needs to be set\n";
   $RRDTOOL_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or warn "Check $file.  Value for \$RRDTOOL_PGM: \"$program\" not executable: $!\n";

   # Verify $SSH_PGM
   $SSH_PGM =~ /not set/ and die "Check $file.  Value for \$SSH_PGM needs to be set\n";
   $SSH_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or warn "Check $file.  Value for \$SSH_PGM: \"$program\" not executable: $!\n";

   # Verify $SCP_PGM
   $SCP_PGM =~ /not set/ and die "Check $file.  Value for \$SCP_PGM needs to be set\n";
   $SCP_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or die "Check $file.  Value for \$SCP_PGM: \"$program\" not executable: $!\n";

   # Verify $TELNET_PGM
   $TELNET_PGM =~ /not set/ and die "Check $file.  Value for \$TELNET_PGM needs to be set\n";
   $TELNET_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or die "Check $file.  Value for \$TELNET_PGM: \"$program\" not executable: $!\n";

   # Verify $TRACE_ROUTE_PGM
   $TRACE_ROUTE_PGM =~ /not set/ and die "Check $file.  Value for \$TRACE_ROUTE_PGM needs to be set\n";
   $TRACE_ROUTE_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or die "Check $file.  Value for \$TRACE_ROUTE_PGM: \"$program\" not executable: $!\n";

   # Verify $W_PGM
   $W_PGM =~ /not set/ and die "Check $file.  Value for \$W_PGM needs to be set\n";
   $W_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or die "Check $file.  Value for \$W_PGM: \"$program\" not executable: $!\n";

   # Verify $ZIP_PGM
   $ZIP_PGM =~ /not set/ and die "Check $file.  Value for \$ZIP_PGM needs to be set\n";
   $ZIP_PGM =~ /$pgm_filter/;
   $program = $1;
   -x $program or die "Check $file.  Value for \$ZIP_PGM: \"$program\" not executable: $!\n";

   # Put certain variables into the environment that are required by child processes
   $ENV{SCP_PGM} = "$SCP_PGM";
   $ENV{HOST} = "$host";

} else {

   # Getting here means no pt.conf file was found.  Create a new one.
   open PT_CONF, ">", "$file" or die "Cannot open $file for writing: $!";
   print PT_CONF << "EOF";
#----- ping_test ($script_ver $script_date) ---------------------------------
# This is the program locations configuration file for the ping_test program.
# Uncomment the lines that correspond to your operating system.  The program 
# locations are based on standard locations, but might need to be changed if 
# your installation is different.  Only change the path to the program. Don't 
# change any arguments on the program lines. They are required for ping_test 
# operation.

# Solaris values
#our \$HOST_PGM ='/usr/ucb/hostname';
#our \$LS_PGM ='/bin/ls';
#our \$MAIL_PGM ='/usr/ucb/mail';
#our \$NETSTAT_PGM ='/usr/bin/netstat -P tcp';
#our \$PING_PGM ="/usr/sbin/ping -s -n ip_address 56 1";
#our \$PS_PGM ='/bin/ps';
#our \$RRDTOOL_PGM ='/usr/bin/rrdtool';
#our \$SCP_PGM ='/opt/bin/scp';
#our \$SSH_PGM ='/opt/bin/ssh';
#our \$TELNET_PGM ='/usr/bin/telnet';
#our \$TRACE_ROUTE_PGM ='/usr/sbin/traceroute -I';
#our \$W_PGM ='/usr/bin/w';
#our \$ZIP_PGM ='/usr/bin/zip';

# Linux 2.x values
#our \$HOST_PGM ='/bin/hostname';
#our \$LS_PGM ='/bin/ls';
#our \$MAIL_PGM ='/bin/mail';
#our \$NETSTAT_PGM ='/bin/netstat -t';
#our \$PING_PGM ="/bin/ping -q -c 1 ip_address";
#our \$PS_PGM ='/bin/ps';
#our \$RRDTOOL_PGM ='/usr/bin/rrdtool';
#our \$SCP_PGM ='/usr/bin/scp';
#our \$SSH_PGM ='/usr/bin/ssh';
#our \$TELNET_PGM ='/usr/bin/telnet';
#our \$TRACE_ROUTE_PGM ='/usr/sbin/traceroute -I';
#our \$W_PGM ='/usr/bin/w';
#our \$ZIP_PGM ='/usr/bin/zip';

# cygwin 1.5.x values assuming standard install to c:\\cygwin
# Note: You need to install the following non base cygwin packages:
#       ping
#       openssh
#       procps
#       zip
#       email
#
# You need to uncomment the correct location of tracert and netstat below
#our \$NETSTAT_PGM ='/c/WINDOWS/system32/netstat';
#our \$NETSTAT_PGM ='/c/WINNT/system32/netstat';
#our \$TRACE_ROUTE_PGM ='/c/WINDOWS/system32/tracert';
#our \$TRACE_ROUTE_PGM ='/c/WINNT/system32/tracert';
#our \$HOST_PGM ='/usr/bin/hostname';
#our \$LS_PGM ='/usr/bin/ls';
#our \$MAIL_PGM ='/usr/bin/email';
#our \$PING_PGM ="/usr/bin/ping -s -n ip_address 56 1";
#our \$PS_PGM ='/usr/bin/procps';
#our \$RRDTOOL_PGM ='/usr/local/rrdtool-1.2.15/rrdtool';
#our \$SCP_PGM ='/usr/bin/scp';
#our \$SSH_PGM ='/usr/bin/ssh';
#our \$TELNET_PGM ='/usr/bin/telnet';
#our \$W_PGM ='/usr/bin/w';
#our \$ZIP_PGM ='/usr/bin/zip';

# Copyright: 2003-2006 by John R Larsen  -  john\@larsen-family.us
# http://pingtest.sourceforge.net
# Released under the same Artistic license as perl

EOF
   close PT_CONF;
   print STDERR "Writing new $file file.  You must edit this file to select your operating system.\n";
   exit 0;
   }

} # setup_env


#------------------------------------------------------------------------------
# sleep_time: Function that adjusts sleep time based on script execution time
# and the time it takes the OS to queue the task and wake it up.
sub sleep_time {
	my $func_name = "sleep_time";
   my $sleep_time;
   my $while_end_epoch;
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside sleep_time\n", __LINE__) }

   # Capture the current epoch time
   $while_end_epoch = time;

   # Increment to the next predicted start time
   $predicted_start_time += $ping_rate;

   # Normally the while loop is supposed to wake up at the predicted start time and do all its processing
   # before the next predicted start time.  If something takes longer, such as a webpage update or some
   # OS delays, the while loop processing could end after the predicted start time.  The goal is to smooth
   # out the while loops so they are periodic.  If the task ends late then the sleep time is set to zero
   # so that the while loop will run almost immediately.  This makes it possible to catch up from a long
   # running time slice.
   if ($while_end_epoch < $predicted_start_time) {
      # Task woke up within the expected time slice. Calculate how much time need to sleep.
      $sleep_time = $predicted_start_time - $while_end_epoch;
   } else {
      # Task didn't wake up within expected time slice.  Run immediately
      # to try to catch up.  It might take several time slices depending
      # on how long the delay was.
      printf ("[%d]$func_name:$$> Task ended %d seconds late\n", __LINE__, ($while_end_epoch - $predicted_start_time));
      $sleep_time = 0;
   }

   # Save the calculated amount of sleep time in case it needs to be displayed later
   my $calculated_sleep_time = $sleep_time;

   # The sleep call returns how many seconds it actually slept.
   my $seconds_slept = sleep $sleep_time;

   # Now check if the time actually elapsed or not 
   if ($seconds_slept < $sleep_time) {
      my $count = 0;
      # Getting here means "sleep" ended early.  Sleep the difference.
      # Stay in this while loop until the time has elapsed. Cygwin and Linux
      # seem to end early at times, especially cygwin.  
      while ($seconds_slept < $sleep_time) {
         $sleep_time -= $seconds_slept;
         $seconds_slept = sleep $sleep_time;
         $count++;
         $early_sleep_exit_cntr++;
      }

      # Log event
      my $date = &get_time;
      printf ("[%d]$func_name:$$> sleep woke up early $count times at %s\n", __LINE__, $date);
   }


	# Output runtime statistics if debugging is turned on
	if ($D & 0x2000) {
		print "--------------------------------------------------\n";
		print "while_start_epoch:      $while_start_epoch\n";
		print "while_end_epoch:        $while_end_epoch\n";
		print "calculated_sleep_time:  $calculated_sleep_time\n";
		print "predicted_start_time:   $predicted_start_time\n";
		print "ping_rate:              $ping_rate\n";
		print "early_sleep_exit_cntr:  $early_sleep_exit_cntr\n";
   }
} # sleep_time


#------------------------------------------------------------------
# test_port:  Function that tests if a connection can be made to a ssh port on a remote computer.
# It uses telnet to connect to the port.  It looks for "Connected to" in the response.
# Arg0: is the host name.      Fatal error if not set.
# Arg1: is the port to test.   Defaults to 22 if not set.
# Arg2: is the timeout value.  Defaults to 30 if not set.
# Returns:  GOOD_PORT if able to connect to port or BAD_PORT if connection is refused or times out
sub test_port {
	my $func_name = "test_port";
   my $pid;
   my $host = $_[0] ? $_[0] : die "host missing: $!";
   my $port = $_[1] ? $_[1] : 22;
   my $timeout = $_[2] ? $_[2] : 30;
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside test_port (host: $host   port: $port   timeout: $timeout)\n", __LINE__) }

   # Use fork so the test_port can be terminated if it takes too long
   TEST_PORT_FORK: {
      if ($pid = fork) {
         # Parent ends up here
         my $counter = 0;
         my $rc_pid;
         # Test child pid every second until the timeout value is reached
         while ($counter < $timeout) {
            if ($pid == (waitpid ($pid, WNOHANG))) {
               return GOOD_PORT;;
            }
            sleep 1;
            $counter++;
         }
         # Getting here means the port test timed out
         kill 9, $pid;
         printf ("[%d]$func_name:$$> test_port timed out (host: $host   port: $port   timeout: $timeout)\n", __LINE__);

         # Increment $bad_port_cntr
         $bad_port_cntr++;
         printf ("[%d]$func_name:$$> $bad_port_cntr bad test ports have occurred to $host port $port\n", __LINE__);
         return BAD_PORT;
      }
      elsif (defined $pid) { # $pid is zero here if defined
         # child ends up here
         # A good telnet connection to an ssh server will have "Connected to" in the data
         my $data = `echo test | $TELNET_PGM $host $port 2>1`;
         if ( $D & 0x80_0000 ) { printf ("[%d]$func_name:$$> Data from telnet connection:\n$data\n", __LINE__) }
         if ( $data =~ /.*Connected to.*/) {
            # Successful connection so exit normally
            exit 0;
         }
         # If unable to connect then sleep longer than timeout so parent kills us
         sleep (10 + $timeout);
      }
      elsif ($! == EAGAIN) {
         # EAGAIN is the supposedly recoverable fork error
         sleep 5;
         redo TEST_PORT_FORK;
      }
      else {
         # weird fork error occurred
         die "Can't fork: $!\n";
      }
   }
} # test_port


#------------------------------------------------------------------
# trace_route:  Function that does traceroute on IP 
# Arg1: Timeout value (default: 15 seconds)
# The route is saved to file $ip.route in the current directory.
sub trace_route {
	my $func_name = "trace_route";
   my $end_time;
   my $timeout = 15;
   $timeout = $_[0] if ($_[0]);
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside trace_route (Timeout: $timeout)\n", __LINE__) }

   # Capture time trace started
   my $start_time = &get_time;

   # Use fork so the traceroute can be terminated if it takes too long
   my $pid;
   FORK: {
      if ($pid = fork) {
         # Parent ends up here
         my $counter = 0;
         my $rc_pid;
         while ($pid != ($rc_pid = waitpid ($pid, WNOHANG))) {
            if (++$counter > $timeout) {
               printf ("%s - [%d]$func_name:$$> traceroute timed out (timeout value: $timeout)\n", $start_time, __LINE__);
               kill 9, $pid;
               last;
            } 
            else {
               sleep 1;
            }
         }
         # The traceroute call keeps going once it is started even if the process that
         # launched it is killed.  Copy the contents of the temp file into $ip.route
         # so start and stop times can be appended without getting overwritten.
         copy ("/tmp/$ip.route", "$ip.route");
         my $end_time = &get_time;
         open IP_ROUTE, ">>", "$ip.route" or die "Cannot open $ip.route for writing: $!";
         print IP_ROUTE "\ntraceroute started: $start_time\n";
         print IP_ROUTE "traceroute ended:   $end_time\n";
         close IP_ROUTE;
      }
      elsif (defined $pid) { # $pid is zero here if defined
         # child ends up here
         system "$TRACE_ROUTE_PGM $ip 2>&1 1>/tmp/$ip.route";
         exit 0;
      }
      elsif ($! == EAGAIN) {
         # EAGAIN is the supposedly recoverable fork error
         sleep 5;
         redo FORK;
      }
      else {
         # weird fork error occurred
         die "Can't fork: $!\n";
      }
   }
} # trace_route


#------------------------------------------------------------------------------
# web_page. Function that creates a web page and copies it somewhere periodically.
# If the file webpage.copy exists then the web page is created and webpage.copy
# is called with the name of the web page in $1.  The webpage.copy file is unique
# and must be self contained to copy the file where it needs to be.  This can
# be to another drive on the LAN or could be using scp.
# Arg0: Forces an update if present
sub web_page {
   my $func_name = "webpage";
   my $current_state;
   my $status;
   my $force_update = $_[0] ? "force_update" : "use_counter";
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside web_page: $webpage_update_cntr\n", __LINE__) }

   # Check if $webpage_remote_dir and $webpage_local_dir are both changed from the default "none"
   return 0 if ($webpage_remote_dir eq "none" && $webpage_local_dir eq "none");

   # Getting here means $webpage_remote_dir or $webpage_local_dir is a path instead of "none"

   # Perform an update if "force_update" is set or if webpage_update_cntr is not greater than 1
   if ($force_update ne "force_update") {
      # Only proceed if web page update counter is greater than 1 meaning an update is due
      if ($webpage_update_cntr > 1) {
         $webpage_update_cntr--;
         return 0;
      } 
      else {
         $webpage_update_cntr = $webpage_update_rate;
      }
   }

   # Getting here means it's time to create a new web page.  Do this in a forked process so that
   # cron, loader, and tunnel will keep going even if something bad happens.  There have been
   # times when OS calls like netstat hang and took down everything.

   # Getting here means we need to fork a process to do the webpage update
   # Use fork so the loader can keep going.
   WEB_PAGE_FORK: {
      if (my $pid = fork) {
         # Parent ends up here.  Simply return.
         return 0;
      }
      elsif (defined $pid) { # $pid is zero here if defined
         # child ends up here
         # Capture the creation time for the webpage
         my $date = &get_time;

         # Make RRD graphs
         &rrd_graph;

         # Determine the current state of the site
         if ($dropped_email_sent == TRUE) {
            $current_state = "DOWN";
         } else {
            $current_state = "UP";
         }

         # A race condition on Cygwin on Win2000 can create an invisible file that can't be overwritten
         # or erased if the unlink command is used.  The work around is to simply open the file for
         # writing twice.  The first time with ">" to simply overwrite the previous contents
         # and then open it again with ">>" so it is appended.
         open IP_HTML, ">", "$ip.html" or die "Cannot open $ip.html for writing: $!";

         # Redirect all output to file handle IP_HTML
         print IP_HTML << "EOF";
         <HTML>
         <HEAD>
            <TITLE>Ping statistics:  $host to $ip</TITLE>
            $webpage_refresh_rate
         </HEAD>
         <BODY>
            <h1> Ping statistics:  $host to $ip</h1>
            <h6> ping_test $script_ver $script_date</h6>
            <h2> Time: $date</h2>
            <h2> Current state: $current_state</h2>
            $include_rrd_graphs
EOF

         # Open it the second time with appending enabled
         open IP_HTML, ">>", "$ip.html" or die "Cannot open $ip.html for writing: $!";

         # Output the overall statistics
         print IP_HTML "<h3> Overall Statistics since $db[TEST_STARTED_YMD] $db[TEST_STARTED_HMS] $db[TEST_STARTED_DOW] </h3>\n";
         print IP_HTML "<pre>\n";
         $status = `cat $ip.stats 2>&1`;
         print IP_HTML "$status";
         print IP_HTML "</pre>\n";

         if (-e "$ip.weeks") {
            # Output the weekly statistics
            print IP_HTML "<h3>Weekly Statistics since $db[TEST_STARTED_YMD] $db[TEST_STARTED_HMS] $db[TEST_STARTED_DOW] </h3>\n";
            print IP_HTML "<pre>\n";
            $status = `cat $ip.weeks 2>&1`;
            print IP_HTML "$status";
            print IP_HTML "</pre>\n";
         }

         if ( -e "$ip.route" ) {
            print IP_HTML "<h3>traceroute from $host to $ip </h3>\n";
            print IP_HTML "<pre>\n";
            $status = `cat $ip.route 2>&1`;
            print IP_HTML "$status";
            print IP_HTML "</pre>\n";
         }

         if (-e "$ip.emails") {
            print IP_HTML "<h3>email log of emails sent </h3>\n";
            print IP_HTML "<pre>\n";
            $status = `cat $ip.emails 2>&1`;
            print IP_HTML "$status";
            print IP_HTML "</pre>\n";
         }

         if ($D & 0x4_0000) {
            print IP_HTML "<!-- <h3>Suppressing w, ps, netstat</h3>\n -->";
         }
         else {
            # Cygwin on XP seems to hang with lots of processes that never terminate.  Mostly associated with
            # procps, w, and perl.  Trying to isolate the problem by disabling those in the webpage.
            # Trying alternate method as discovered in ssh_tunnel.pl that seemed to avoid the problem.
            print IP_HTML "<h3>Current users, uptime, and load average</h3>\n";
            print IP_HTML "<pre>\n";
            $status = `$W_PGM 2>&1`;
            print IP_HTML "$status";
            print IP_HTML "</pre>\n";

            print IP_HTML "<h3>Current processes</h3>\n";
            print IP_HTML "<pre>\n";
            $status = `$PS_PGM -edf 2>&1`;
            print IP_HTML "$status";
            print IP_HTML "</pre>\n";

            print IP_HTML "<h3>Current netstat</h3>\n";
            print IP_HTML "<pre>\n";
            $status = `$NETSTAT_PGM 2>&1`;
            print IP_HTML "$status";
            print IP_HTML "</pre>\n";
         }

         print IP_HTML "<h3>crontab file</h3>\n";
         print IP_HTML "<pre>\n";
         $status = `crontab -l 2>&1`;
         print IP_HTML "$status";
         print IP_HTML "</pre>\n";

         print IP_HTML "<h3>$ip.conf file</h3>\n";
         print IP_HTML "<pre>\n";
         $status = `cat $ip.conf 2>&1`;
         print IP_HTML "$status";
         print IP_HTML "</pre>\n";

         print IP_HTML "</BODY> </HTML>\n";

         # All done building the message so close the file handle
         close IP_HTML;

         # Make a local copy if $webpage_local_dir is not "none"
         if ($webpage_local_dir ne "none") {
            copy "$ip.html", "$webpage_local_dir/$webpage_filename.html";
         }

         # Only do the webserver update if $webpage_remote_dir isn't "none"
         exit 0 if ($webpage_remote_dir eq "none");
         
         # Only attempt an scp copy if the port checks out good
         my $port_state = &test_port ($webserver_domain_name, $webserver_domain_port);
         if ($port_state != GOOD_PORT) {
            printf ("[%d]$func_name:$$> Webserver $webserver_domain_name port $webserver_domain_port is bad\n", __LINE__);
            exit 1;
         }

         my $verbose = ($D & 0x2_0000) ? "-v" : "";
         my $redirection = ($D & 0x2_0000) ? "" : "2>/dev/null";
         $rc = system "$SCP_PGM -p $verbose -F $working_dir/$ip.ssh $ip.html webpage:$webpage_remote_dir/$webpage_filename.html $redirection";
         if ($rc) {
            printf ("[%d]$func_name:$$> Webpage copy failed to $webserver_domain_name\n", __LINE__);
            exit 1;
         }
         else {
            printf ("[%d]$func_name:$$> Webpage copied to $webserver_domain_name  $webpage_remote_dir/$webpage_filename.html\n", __LINE__);
            exit 0;
         }
      }
      elsif ($! == EAGAIN) {
         # EAGAIN is the supposedly recoverable fork error
         sleep 5;
         redo WEB_PAGE_FORK;
      }
      else {
         # weird fork error occurred
         die "Can't fork: $!\n";
      }
   }
} # web_page


#------------------------------------------------------------------------------
# write_pt_db:  Function that writes the pt database
sub write_pt_db {
   open IP_DB, ">", "$$.db" or die "Cannot open file $$.db: $!";
   print IP_DB "@db";
   close IP_DB;
   # Read the temp file back and compare it with @db values to be
   # sure the file was created correctly before overwriting the old file.
   open IP_DB, "<", "$$.db" or die "Cannot open file $$.db: $!";
   my $test_db_file = <IP_DB>;
   my $test_db_pgm = "@db";
   if ($test_db_file eq $test_db_pgm) {
      rename "$$.db", "$ip.db";
   } else {
      warn "Database not saved.  Error saving database to file.\n";
      unlink "$$.db";
   }
} # write_pt_db


#------------------------------------------------------------------------------
# write_stats_file:  Function that writes the stats file
# 
sub write_stats_file {
my $host_to_ip = "$host to $ip (pid: $$)";

format STATS =
------------------------------------------------------- @<<<< @<<<<<<<<< -------------
                                                        $script_ver, $script_date
Ping Test:           @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
                     $host_to_ip
Total pings:         @<<<<<<
                     $db[0]
Total good pings:    @<<<<<<
                     $db[1]
Total dropped pings: @<<<<<<
                     $db[2]
Percent loss:        @<<<< %
                     $db[3]
Minimum ping time:   @<<<<<<   ms on @<<<<<<<<<< @<<<<<<< @<<
                     $db[4],         $db[5],     $db[6],  $db[7]
Average ping time:   @<<<<<<<< ms
                     $db[8]
Maximum ping time:   @<<<<<<<< ms on @<<<<<<<<<< @<<<<<<< @<<
                     $db[9],         $db[10],    $db[11], $db[12]
Total emails sent:   @<<<<<<
                     $db[13]
Last email sent:     @<<<<<<<<<< @<<<<<<< @<<
                     $db[14],    $db[15], $db[16]
Test started:        @<<<<<<<<<< @<<<<<<< @<<
                     $db[17],    $db[18], $db[19]
Threshold:           @<< dropped packets before email sent
                     $threshold
Ping rate:           @<< seconds between pings
                     $ping_rate
Email address:       @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
                     $email_address

Week: @<                 Sun     Mon     Tue     Wed     Thu     Fri     Sat    Week
      $db[20]
======================================================================================
@<<<<<<<< - Good:    @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>>
$db[21],             $db[22],$db[23],$db[24],$db[25],$db[26],$db[27],$db[28],$db[29]
@<<<<<<<< - Dropped: @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>>
$db[21],             $db[30],$db[31],$db[32],$db[33],$db[34],$db[35],$db[36],$db[37]
@<<<<<<<< - Loss %:    @>>>>   @>>>>   @>>>>   @>>>>   @>>>>   @>>>>   @>>>>   @>>>>
$db[21],             $db[38],$db[39],$db[40],$db[41],$db[42],$db[43],$db[44],$db[45]
@<<<<<<<< - Emails:  @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>>
$db[21],             $db[46],$db[47],$db[48],$db[49],$db[50],$db[51],$db[52],$db[53]
@<<<<<<<< - Min-ms:  @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>>
$db[21],             $db[54],$db[55],$db[56],$db[57],$db[58],$db[59],$db[60],$db[61]
@<<<<<<<< - Avg-ms:  @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>>
$db[21],             $db[62],$db[63],$db[64],$db[65],$db[66],$db[67],$db[68],$db[69]
@<<<<<<<< - Max-ms:  @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>> @>>>>>>
$db[21],             $db[70],$db[71],$db[72],$db[73],$db[74],$db[75],$db[76],$db[77]
---- @<<<< @<<<<<<<<< ------------- ping rate: @<< ----- threshold: @< ---------------
     $script_ver, $script_date,               $ping_rate,          $threshold
.

open STATS, ">", "$ip.stats" or die "Cannot open $ip.test: $!";
write STATS;
close STATS;
} # write_stats_file


#------------------------------------------------------------------------------
# yes_or_no: Subroutine that uses $_[0] as a prompt and returns YES and or NO constants
sub yes_or_no {
   my $func_name = "yes_or_no";
   my $input = '';
   my $prompt = $_[0];
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside yes_or_no\n", __LINE__) }

   SWITCH: {
      printf STDERR ("%s? (yes or no): ", $prompt);
      chomp($_ = <STDIN>);
      if (/^yes\b|^y\b/i){return YES;}
      if (/^no\b|^n\b/i){return NO;}
      printf STDERR ("Invalid choice: %s\n", $_);
      redo SWITCH;
   }
} # yes_or_no


#------------------------------------------------------------------------------
# main: 
sub main {
   my $pt_pid;
   my $dow;
   my $status;
   my $func_name = "main";

   &process_cmd_line;

   # At this point the $ip variable must be set either from the command line or
   # from the symbolic link.  Check for valid $ip before continuing.
   if ( $ip eq "not_set" ) {
      print STDERR "ERROR:  IP or domain name missing.";
      print STDERR "Enter \"./pt.pl -h\" for help  (ping_test $script_ver $script_date)\n";
      exit 1;
   }

   # Need to select log destination now that $ip is set
   &set_log_dest;

   # Read in the configuration file for the IP being tested
   &read_ip_conf;

   # Need to modify global PING_PGM with $ip
   $PING_PGM =~ s/ip_address/$ip/;

   # If the program was invoked as "pt.pl" that means it should do cron processing
   if ( $pgm eq "pt.pl" ) {
      &cron_processing;
   }

   # Read in the $ip.db file to initialize the @db array with saved values
   &read_pt_db;

   # These variables don't get changed by the user
   $dropped_cntr = 0;
   $dropped_email_sent = FALSE;

   # Exit if there is an active PID
   $rc = &pid_active;
   if ( $rc ) {
      printf ("$host to $ip: Active pid $rc found. Exiting\n");
      printf STDERR ("$host to $ip: Active pid $rc found\n");
      printf STDERR ("Use: pt.pl $ip -k   to terminate\n");
      exit 1;
   } else {
      # Since the stored PID isn't active, save the current PID and program path as the active PID
      &pid_save;
   }

   # Add entry to existing logfile
   &log_entry;

   # Setup week pointers into the statistics array
   my $week_good_idx = 22;
   my $week_dropped_idx = 30;
   my $week_loss_idx = 38;
   my $week_emails_idx = 46;
   my $week_min_idx = 54;
   my $week_avg_idx = 62;
   my $week_max_idx = 70;
   
   # Do a traceroute so the file has an updated route
   &trace_route ($traceroute_timeout);
   
   # Force a webpage update
   &web_page ("force_update");

   # Initialize the predicted start time so sleep_time will have sensible number
   $predicted_start_time = time;

   # Save pt's pid into a file.  The value of $$ is compared to this saved value each time through
   # the main loop.  If another instance of pt somehow gets started, it will overwrite this file
   # and other instances of pt will exit.  This insures only one pt runs at a time.  Cygwin seems
   # to have problems with this more than Solaris and Linux.
   open IP_PID, ">", "$ip.pid" or die "Cannot open file $ip.pid: $!";
   print IP_PID "$$\n";
   print IP_PID "$0\n";
   close IP_PID;

   # Stay in this while loop forever
   while (FOREVER) {
      if ($D & 0x4) {printf ("[%d]$func_name:$$> Inside main while loop\n", __LINE__) }
      my $while_start_date = &get_time;
      $while_start_epoch = time;

      # Display the environment if debugging enabled
      if ($D & 0x10) {&display_env}

      # Verify this process has same pid as in file $ip.pid.  Exit if different.
      $pt_pid = &get_pt_pid;
      if ($$ != $pt_pid) {
         my $date = &get_time;
         printf ("%s - pt: current pid $$ doesn't equal saved pid $pt_pid.  Exiting.\n", $date);
         exit 1;
      }

      # See if debug level has changed
      &set_debug_output;

      # Write to the heartbeat file so cron can tell the script is still alive
      open IP_HEARTBEAT, ">", "$ip.heartbeat" or die "Cannot open file $ip.heartbeat: $!";
		print IP_HEARTBEAT "RUNNING";
      close IP_HEARTBEAT;

      # The logfile processing needs to be done here.  Add it later.
      &logfile;

      # Increment the total pings count
      $db[TOTAL_PINGS]++;

      # Perform the ping
      $ping_time = &ping_ip;

      # See if we're into a new week before setting up indices
      &check_week;

      # Get the day of the week and setup indices into the statistics array
      $dow = `date '+%w'`;
      my $day_good_idx = $dow + 22;
      my $day_dropped_idx = $dow + 30;
      my $day_loss_idx = $dow + 38;
      my $day_emails_idx = $dow + 46;
      my $day_min_idx = $dow + 54;
      my $day_avg_idx = $dow + 62;
      my $day_max_idx = $dow + 70;

      # If ping_test debug is turned on then force the value of $ping_time
      if ($D & 0x80) {
         if ($toggle_cnt == 0) {
            $toggle_cnt = 5;
            if ($test_ping_status eq "ping_good") {
               printf ("[%d]$func_name:$$> Forcing \$ping_time to -1\n", __LINE__);
               $test_ping_status = "ping_bad";
               $ping_time = -1;
            } else {
               printf ("[%d]$func_name:$$> Using actual \$ping_time\n", __LINE__);
               $test_ping_status = "ping_good";
            }
         } else {
            $toggle_cnt--;
            if ($test_ping_status eq "ping_bad") {
               $ping_time = -1;
            }
         }
      }

      # Process the results of the ping
      if ($ping_time == -1) {
         # Increment the bad ping counters
         $db[DROPPED_PINGS]++;
         $db[$day_dropped_idx]++;
         $db[WEEK_DROPPED_IDX]++;

      } else {
         # Increment the good ping counters
         $db[GOOD_PINGS]++;
         $db[$day_good_idx]++;
         $db[WEEK_GOOD_IDX]++;

         # Replace overall minimum ping time if this one was faster
         if ($ping_time < $db[MIN_PING_TIME]) {
            $db[MIN_PING_TIME] = $ping_time;
            push my @time_array,  split / /, $time_of_ping;
            $db[MIN_PING_DATE_YMD] = $time_array[0];
            $db[MIN_PING_DATE_HMS] = $time_array[1];
            $db[MIN_PING_DATE_DOW] = $time_array[2];
         }

         # Replace daily minimum ping time if this one was faster
         if ($db[$day_min_idx] == 0) {
            $db[$day_min_idx] = 10_000;
         }
         if ($ping_time < $db[$day_min_idx]) {
            $db[$day_min_idx] = $ping_time;
         }

         # Replace weekly minimum ping time if this one was faster
         if ($db[WEEK_MIN_IDX] == 0) {
            $db[WEEK_MIN_IDX] = 10_000;
         }
         if ($ping_time < $db[WEEK_MIN_IDX]) {
            $db[WEEK_MIN_IDX] = $ping_time;
         }

         # Replace overall maximum ping time if this one was longer
         if ($ping_time > $db[MAX_PING_TIME]) {
            $db[MAX_PING_TIME] = $ping_time;
            push my @time_array,  split / /, $time_of_ping;
            $db[MAX_PING_DATE_YMD] = $time_array[0];
            $db[MAX_PING_DATE_HMS] = $time_array[1];
            $db[MAX_PING_DATE_DOW] = $time_array[2];
         }

         # Replace daily maximum ping time if this one was faster
         if ($ping_time > $db[$day_max_idx]) {
            $db[$day_max_idx] = $ping_time;
         }

         # Replace weekly maximum ping time if this one was faster
         if ($ping_time > $db[WEEK_MAX_IDX]) {
            $db[WEEK_MAX_IDX] = $ping_time;
         }

         # Add ping time to daily sum, calculate the average, and save it to the stats array
         $db[SUMS_SUN + $dow] += $ping_time;
         $db[$day_avg_idx] = $db[SUMS_SUN + $dow] / $db[$day_good_idx];

         # Add ping time to weekly sum, calculate the average, and save it to the stats array
         $db[SUMS_WEEK] += $ping_time;
         $db[WEEK_AVG_IDX] = $db[SUMS_WEEK] / $db[WEEK_GOOD_IDX];

         # Add ping time to overall sum, calculate the average, and save it to overall average $AVG_PING_TIME
         $db[SUMS_OVERALL] += $ping_time;
         $db[AVG_PING_TIME] = $db[SUMS_OVERALL] / $db[GOOD_PINGS];
      }
      
      # Calculate the daily, weekly, and overall loss percentages 
      $db[$day_loss_idx] = $db[$day_dropped_idx] / ($db[$day_dropped_idx] + $db[$day_good_idx]) * 100;
      $db[WEEK_LOSS_IDX] = $db[WEEK_DROPPED_IDX] / ($db[WEEK_DROPPED_IDX] + $db[WEEK_GOOD_IDX]) * 100;
      $db[PERCENT_LOSS] = $db[DROPPED_PINGS] / $db[TOTAL_PINGS] * 100;

      # Send email after $threshold dropped pings in a row.  Send another email immediately after a successful ping.
      if ($ping_time == -1) {
         # Getting here means the ping failed.
         # Check if a "down" email needs to be sent
         $dropped_cntr++;
         if (($dropped_cntr > $threshold) && ($dropped_email_sent == FALSE)) {
            $dropped_cntr = $threshold;
            $dropped_email_sent = TRUE;
            $db[$day_emails_idx]++;
            $db[WEEK_EMAILS_IDX]++;
            &send_email ("Down");
         }
      } else {
         # Getting here means the ping was good.
         $dropped_cntr--;
         # Don't let dropped counter go below zero
         if ($dropped_cntr < 0) {
            $dropped_cntr = 0;
         }

         # If down email has been sent, clear counter, flag, and send an "up" email
         if ($dropped_email_sent == TRUE) {
            $dropped_cntr=0;
            $dropped_email_sent = FALSE;
            $db[$day_emails_idx]++;
            $db[WEEK_EMAILS_IDX]++;
            &send_email ("Up");
         }
      }

      # Check if time to create and transfer a new web page
      &web_page;

      # Output the $ip.stats file if debugging is turned on
      if ($D & 0x100) {
         printf ("\n[%d]$func_name:$$> Contents of $ip.stats\n\n", __LINE__);
         $status = `cat $ip.stats`;
         print "$status";
      }

      # Output the $ip.weeks file if debugging is turned on
      if ($D & 0x200) {
         if (-s "$ip.weeks") {
            printf ("\n[%d]$func_name:$$> Contents of $ip.weeks\n\n", __LINE__);
            $status = `cat $ip.weeks`;
            print "$status";
         }
      }

      # Update and output the $ip.route file if debugging is turned on
      if ($D & 0x400) {
         &trace_route ($traceroute_timeout);
         printf ("\n[%d]$func_name:$$> Contents of $ip.route\n", __LINE__);
         $status = `cat $ip.route`;
         print "$status";
      }

      # Sleep awhile
      &sleep_time;

      # Try updating the RRD database
      &rrd_update ($ping_time, $db[$day_loss_idx], $db[WEEK_LOSS_IDX], $db[PERCENT_LOSS]);

      &write_stats_file;
      &write_pt_db;

      # Cleanup any zombie processes 
      &reaper;

   } # while (FOREVER)

} # main


# This is where everything actually starts happening
&set_working_dir;
&get_version;

# Check if -hd, -hs or -h on command line and display them even $ip.conf or pt.conf
# haven't been created or modified.  Also check if summary pages should be made.
$rc = GetOptions ( 
   'h|help' => sub {&display_help}, 
   'hd' => sub {&display_debug_help},
   'hs' => sub {&display_setup_help},
   'install' => sub {&install},
   'ms=i' => sub {&make_summary_pages ("$_[1]")},
);

&setup_env;
&init_debug;
&main;

# Cleanup before exiting
close LOG;