#!/bin/env perl -w
require 5.8.8;

=information

Script Name             : monitor_jobs.pl
Author                  : Sanjog Sahu 
Date of Modification    : 23rd, March, 2013. (v1.0.0)


Operating System(s)     : Linux

Description             : Monitors Jobs and sendout alert/stausmails for set of jobs


Execution Method        : 

 monitor_jobs.pl -d 20100215

=cut


use strict;
use warnings;
use Time::Local;
use DateTime::Duration;
use IO::File;
use Data::Dumper;
use FeedUtil;
use Env;
use Getopt::Std;

$|=1;

$debug=1;	#set to 1 if Want to Print all output check on screen 

#####################################################################
#Variable Declaration 
#####################################################################

my %opt;
my $opt=getopts('c:d:m:r:s:h', \%opt);
my $cfg_dir=$CFGROOT;

my $config_file=$cfg_dir . "/monitor_jobs.cfg";
writeLog("info","Job Monitor Config : $config_file");
my $mail_config=$cfg_dir. "/send_mail_monitor_job.xml";
writeLog("info","Job Monitor Mail Config : $mail_config");

my $num_iter=240;
my $sleep=180;
my $amber=10;
my $status_email_rundate;
my $exit=0;
my $REGION='ALL';
my $search='ALL';
my $sla_pending=0;

my %JOB_ERR;
my %JOB_INFO;
my %JOB_DEPENDS_INFO;
my %JOB_DEPENDS_INFO_CHK;

my %SLA_INFO;
my %JOB_MONITOR;
my %FINAL_INFO;
my %DONE=();
my %DEPEND_ADD=();
my %BODY_FILE;

my %JOB_BAD_STATUS=
(
'IN' => 1,
'FA' => 1,
'TE' => 1,
'PE' => 1,
'ST' => 1,
'RE' => 1,
'OI' => 1,
'OH' => 1,
'QU' => 1,
'AC' => 1,

);


&set_param();
writeLog("info","Check Job Rundate : $rundate");
writeLog("info","Status Email Effective Date : $status_email_rundate");


my $MAIL_LOG_FILE=$LOGROOT."/autosys/"."Monitor_Job_email.log.$rundate";
my $mail_body_file=$LOGROOT."/autosys/"."Monitor_Job_email_body_file.html";
writeLog("default","Job Monitor Maliling Log File : $MAIL_LOG_FILE");
writeLog("default","Job Monitor Body File : $mail_body_file");

&parse_config();
&populate_info();

print "-------------CONFIG_INFO-----------\n";
print "\n~~~~~~~~~~~ JOB_INFO ~~~~~~~~~~~~\n\n";
print Dumper(\%JOB_INFO);

print "\n~~~~~~~~~~~ JOB_DEPENDS_INFO ~~~~~~~~~~~~\n\n";
print Dumper(\%JOB_DEPENDS_INFO);

print "\n~~~~~~~~~~~ JOB_DEPENDS_CHECK_INFO ~~~~~~~~~~~~\n\n";
print Dumper(\%JOB_DEPENDS_INFO_CHK);

print "\n~~~~~~~~~~~ SLA_INFO ~~~~~~~~~~~~\n\n";
print Dumper(\%SLA_INFO);

print "\n~~~~~~~~~~~ BODY_FILE ~~~~~~~~~~~~\n\n";
print Dumper(\%BODY_FILE);

print "\n~~~~~~~~~~~ JOB_MONITOR ~~~~~~~~~~~~\n\n";
print Dumper(\%JOB_MONITOR);
print "------------CONFIG_INFO_END-----------\n\n";

my @BODY_NOT_FOUND;

	foreach my $body (keys %BODY_FILE)
	{
		if( -f $BODY_FILE{$body})
		{
		writeLog("info", "Body File Found On Disc [$BODY_FILE{$body}]");
		}
		else
		{
		writeLog("err", "Body File NOT Found On Disc [$BODY_FILE{$body}]");
		push(@BODY_NOT_FOUND,$BODY_FILE{$body});
		}
	}

	if(scalar @BODY_NOT_FOUND > 0)
	{
	writeLog("err", "Following Body File NOT Found On Disc :");
	print "-------------------------------------------------------\n";
	print join("\n",@BODY_NOT_FOUND);
	print "\n-------------------------------------------------------\n";
	exit 1;
	}

	for (my $i=1; $i<=$num_iter ; $i++)
	{
AGAIN:
	print "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n";
	&writeLog("dum","");
	&writeLog("ITERATION","Count [$i]");
	print "-----------------JOBS_DONE------------------\n";
	print Dumper(\%DONE);
	print "--------------------------------------------\n";

	print "-------------JOBS_TO_MONITOR----------------\n";
	print Dumper(\%JOB_MONITOR);
	print "--------------------------------------------\n";
	
	my @JOBS=keys (%JOB_MONITOR);
	print "---------JOB_DEPEND_INFO (PENDING)----------\n";
	print Dumper(\%JOB_DEPENDS_INFO);
	print "--------------------------------------------\n";
	print "Job Monitor Maling Log File : $MAIL_LOG_FILE\n";
	print "--------------------------------------------\n";
	
	print "#############################################################################\n\n";

	my $curr_time=`date +"%d/%m/%Y %H:%M:%S"`;
	chomp $curr_time;
	$sla_pending=0;
		
		if($i > 1)
		{
		&sla_check($curr_time);
		}

		if(scalar keys %JOB_MONITOR == 0 && scalar keys %JOB_DEPENDS_INFO == 0)
		{
		&writeLog("info","No Jobs to Monitor...");
		last;
		}

	my $cnt="01";
		foreach my $job_att (@JOBS)
		{
		my @arr=split(/\|/,$job_att);
		print "****** Checking Job [$cnt] -> [$arr[2]] ******\n";
		writeLog("Job_check_Attribute>","$job_att");
			if($JOB_MONITOR{$job_att} eq 'start')
			{
			my $job_att_ref=&check_attr($job_att,$curr_time);
			print Dumper($job_att_ref);
				if(defined $$job_att_ref{$job_att}{"JOB_OK"})
				{
					if($$job_att_ref{$job_att}{"JOB_OK"} eq 'YES' && $$job_att_ref{$job_att}{"STATUS"} eq "SU")
					{
					my $add=0;
					%FINAL_INFO=%{$job_att_ref};
					
					$DONE{"$arr[0]|$arr[1]|$arr[2]"}="START: $$job_att_ref{$job_att}{START_DATE} $$job_att_ref{$job_att}{START_TIME} ~ END: $$job_att_ref{$job_att}{END_DATE} $$job_att_ref{$job_att}{END_TIME} ~ STATUS: $$job_att_ref{$job_att}{STATUS}";
						#print "JOB---$job_att---\n";
						
						foreach my $job_dep (keys %JOB_DEPENDS_INFO)
						{
						my $val=$JOB_DEPENDS_INFO{$job_dep};
						my ($jb,$run,$buffer)=split(/~/,$val);
						#print "DEPEND:--$job_dep,DEPEND_VAL:--$jb--,RUN:$run,BUFFER:$buffer--\n";
							if(defined $DONE{$jb} && !defined $DEPEND_ADD{$job_dep} )
							{
						#	print "+++++Taken For Monitor: $job_dep\n";
							my $end_date=$FINAL_INFO{$job_att}{'END_DATE'};
							my $end_time=$FINAL_INFO{$job_att}{'END_TIME'};
							## IF job start time is specifiled and also it is a dependent job
								unless(defined $JOB_INFO{$job_dep}{'JOB_START'})
								{
								my $job_start_ts="$end_date $end_time";
								#print "----$job_start_ts,$run\n";
								my $job_end_ts=addTime($job_start_ts,$run);
								my ($date,$ts)=split(/\s+/,$job_end_ts);

								$JOB_MONITOR{"$job_dep|$end_time|$ts|$buffer"}='start';
								}
								else
								{
								my $job_start_ts="$end_date $JOB_INFO{$job_dep}{'JOB_START'}";
								#print "----$job_start_ts,$run\n";
								my $job_end_ts=addTime($job_start_ts,$run);
								my ($date,$ts)=split(/\s+/,$job_end_ts);
								$JOB_MONITOR{"$job_dep|$JOB_INFO{$job_dep}{'JOB_START'}|$ts|$buffer"}='start';
								}
							$DEPEND_ADD{$job_dep}=1;
							delete $JOB_DEPENDS_INFO{$job_dep};
							$add=1;
						#	print "Jobs to monitor:\n";
						#	print Dumper(\%JOB_MONITOR);
						#	print "\n";
							}
							else
							{
						#	print "~~~~NOT Taken For Monitor: $job_dep Depend_add hash\n";
						#	print Dumper(\%DEPEND_ADD);
							print "";
							}
						}
					#print "$job_att YES\n";	
					
					delete $JOB_MONITOR{$job_att};
						if($add)
						{
						goto AGAIN;
						}
					}
					else
					{
					writeLog("info","[$job_att] -> Job [Start Time Not Met | Running State | Didn't kicked off]");
					}
				}
				else
				{
				print "Check More on Job Conditions and Fix\n";
				exit 1;
				}
			}
		
		print "\n#############################################################################\n\n";
		$cnt++;
		}
		
		if($i > 1)
		{
		&sla_check($curr_time);
		}
		
	print "[$i] Sleep $sleep Sec...\n";
	sleep $sleep;
	}

	if(scalar keys %JOB_MONITOR > 0)
	{
	$exit=1;
	&writeLog("err","One Or More Jobs Still Need to be Monitored but Iteration Finished (Please Increase the Iteration in the Config File [$config_file]");
	exit $exit;
	}
	else
	{
	&writeLog("END","$0 Finished...");
	exit 0;
	}


#__END__
#####################################################################


sub sla_check
{
#print "inside SLAAAAAAAAAAA\n";
##SLA Check
my $curr_time=shift;			
	foreach my $sla (keys %SLA_INFO)
	{
	my @val=split(/~/,$SLA_INFO{$sla});
	my $run_dt=&get_format_date($rundate,'yyyymmdd','dd/mm/yyyy');
	my $sla_ts="$run_dt"." "."$sla";
	my $diff_sla=&tdiff($curr_time,$sla_ts);
	#print "CHECK_SLA=$sla==$diff_sla====\n";
		if ($diff_sla < 0)
		{
		my $done=1; 
			foreach my $k (@val)
			{
				unless(defined $DONE{$k})
				{
				$done=0;
				}
			}

			if($done)
			{
			#print "inside DONEEEEEE\n";	
			my $already_sent_flg=&check_log($MAIL_LOG_FILE,"$SLA_INFO{$sla}|STATUS");
			
				unless($already_sent_flg)
				{
				
				my ($reg,$ty,$jb)=split(/\|/,$val[0]);
				my $feed=$reg."__".$ty;
				#print "FEDDDDDDDD $feed\n";
				my $body_file;
					if(defined $BODY_FILE{$feed})
					{
					$body_file=$BODY_FILE{$feed};
					}
					else
					{
					writeLog("err","Body File for [$feed] Not defined in Config file [$config_file]");
					exit 1;
					}
				writeLog("info","Job Done -> SLA ($sla) [Sending Mail to Users]");
				&send_mail($feed,$reg,"$SLA_INFO{$sla}|STATUS")
				}
				else
				{
				writeLog("info","Job Already Done -> SLA ($sla) [Already Sent Mail to Users]");
				}
			}
			else
			{
			#print "__________operate mail\n";
			my $already_sent_flg=&check_log($MAIL_LOG_FILE,"$SLA_INFO{$sla}|operate");
				unless($already_sent_flg)
				{
				my $mbh=IO::File->new("$mail_body_file","w")|| die "Can't open the File [$mail_body_file] in Write mode : $!\n";;
				$mbh->print("One or more Job is not completed or SLA Crossed, Please Check...<br><br>\n");
				$mbh->print("<TABLE border='2' cellpadding='2' style='border-collapse: collapse' bordercolor='#111111' width='95%' ><TR bgcolor='#FA8258' ><td>Region</td><td>Type Process</td><td>Job Name</td><td>Status</td></TR>\n");
				my ($rr,$tt);
					foreach my $key (@val)
					{
					my($reg,$typ,$job_to_sent)=split(/\|/,$key);
					my ($res,$status, $st_date, $st_time, $fin_date,$fin_time)=&get_job_status($job_to_sent);
					$mbh->print("<tr><td>$reg</td><td>$typ</td><td>$job_to_sent</td><td>$res</td></TR>\n");
					$rr=$reg;
					$tt=$typ;
					}
				$mbh->print("</TABLE>\n");
				writeLog("wrn","Crossed SLA -> SLA ($sla) [Sending Mail to Operate]");
				&send_mail('job_alert_operate',"\'Crossed SLA [$rr $tt]\'","$SLA_INFO{$sla}|operate")
				#send operate mail
				#append the log (OPeRATE)
				}
				else
				{
				writeLog("wrn","Already Crossed SLA -> SLA ($sla) [Already Sent Mail to Operate]");
				}
			}
		}
#####################################################################
		else
		{
			# Monitor should wait till SLA time even if the job finished
			if($sla_pending == 0)
			{
			$sla_pending=1;
			}

		#if jobs already completed before the SLA 
		my $done=1; 
			foreach my $k (@val)
			{
				unless(defined $DONE{$k})
				{
				$done=0;
				}
			}
			if($done)
			{
			#print "inside DONEEEEEE\n";	
			my $already_sent_flg=&check_log($MAIL_LOG_FILE,"$SLA_INFO{$sla}|STATUS");

				unless($already_sent_flg)
				{
				
				my ($reg,$ty,$jb)=split(/\|/,$val[0]);
				my $feed=$reg."__".$ty;
				#print "FEDDDDDDDD $feed\n";
				my $body_file;
					if(defined $BODY_FILE{$feed})
					{
					$body_file=$BODY_FILE{$feed};
					}
					else
					{
					writeLog("err","Body File for [$feed] Not defined in Config file [$config_file]");
					exit 1;
					}
				writeLog("info","Job Done -> SLA ($sla) [Sending Mail to Users]");
				&send_mail($feed,$reg,"$SLA_INFO{$sla}|STATUS")
				}
				else
				{
				writeLog("info","Job Already Done -> SLA ($sla) [Already Sent Mail to Users]");
				}
			}
		}
	}
}

#####################################################################
sub send_mail
{
my($feed,$sub,$key)=@_;
my $res=system("$ENV{SCRIPTSROOT}/util/runperl.csh /apps/rvg/bin/util/send_mail_util.pl -f $feed -c $mail_config -d $status_email_rundate -s $sub");
		
	if($res ==0)
	{
	writeLog("info","Mail Sent Successfully");
	append_log_file('append',"$key");
	}
	else
	{
	writeLog("err","Failed to Send Mail");
	exit 1;
	}
}

#####################################################################

sub set_param
{
	unless($opt)
	{
	&usage();
	exit(1);
	}

	if($opt{'h'})
	{
	&usage();
	exit(0);
	}

$rundate=$opt{'d'};
	unless(defined $rundate)
	{
	$rundate=`date +%Y%m%d`;
	chomp $rundate;
	}

$status_email_rundate=$opt{'m'};

	unless(defined $status_email_rundate)
	{
	$status_email_rundate=$rundate;
	}

	if($rundate !~/^\d{8}$/)
	{
	&writeLog("err", "Incorrect RunDate : $rundate");
	&usage();
	exit(1);
	}

	if(defined $opt{'c'})
	{
	$config_file=$opt{'c'};
	}

	unless(-f $config_file)
	{
	&writeLog("err", "Couldn't Find Config File : $config_file");
	&usage();
	exit(1);
	}

	if(defined $opt{'r'})
	{
	$REGION=$opt{'r'};
	}

	if(defined $opt{'s'})
	{
	$search=$opt{'s'};
	}

}

#####################################################################

sub parse_config
{
open(CFG,$config_file) || die "can't open the File [$config_file] for reading :$!\n";
#JOB=JobName ~ START=HH:MM:SS ~ END=HH:MM:SS ~ START_BUFFER=MIN ~ STATUS_SLA=HH:MM:SS ~ JOB_DEPEND=JOBNAME ~ JOB_DEPEND_RUN_TIME=MIN

	foreach my $cfg (<CFG>)
	{
	$cfg=&trim($cfg);

		next if($cfg=~ /^$|^\#/);
	
		if($cfg=~ /\*\*NUMER_OF_ITERATION\s*=\s*(\S+)/)
		{
		$num_iter=$1;
		}

		if($cfg=~ /\*\*SLEEP_SEC\s*=\s*(\S+)/)
		{
		$sleep=$1;
		}

		if($cfg=~ /\*\*MAIL_CFG\s*=\s*(\S+)/)
		{
		$mail_config=$1;
		}

		if($cfg=~ /\*\*AMBER\s*=\s*(\S+)/)
		{
		$amber=$1;
		}
	
		if($cfg=~ /^\-\-/)
		{
			if($cfg=~ /^\-\-(\w+)\s*=\s*(\S+)/)
			{
			$BODY_FILE{$1}=$2;
			}
		}

		if($cfg !~ /^\*\*|\-\-/)
		{
		my @CFG_PARAM=split(/~/,$cfg);
		my ($job,$region,$type);
			foreach my $fld (@CFG_PARAM)
			{
			$fld=&trim($fld);	
				if($fld=~ /JOB\s*=\s*(\S+)/)
				{
				$job=$1;
				last;
				}
			}
			foreach my $fld (@CFG_PARAM)
			{
			$fld=&trim($fld);	
				if($fld=~ /REGION\s*=\s*(\S+)/)
				{
				$region=$1;
				last;
				}
			}
	    
			foreach my $fld (@CFG_PARAM)
			{
			$fld=&trim($fld);	
				if($fld=~ /TYPE\s*=\s*(\S+)/)
				{
				$type=$1;
				last;
				}
			}
	    
			unless (defined $job || defined $region || defined $type )
			{
			writeLog("err","Incorrect config, either JOB=|REGION=|TYPE= Not defined");
			print "==>$cfg<==\n";
			exit 1;
			}
			
			if($REGION ne 'ALL')
			{
			next if ($REGION ne $region);
			}
			
			if($search ne 'ALL')
			{
			my @keys=split(/\|/,$search);
			my $found=0;	
				foreach my $kk (@keys)
				{
					if($kk eq $type)
					{
					$found=1;
					}
				}	
			next if($found == 0);
			}

			foreach my $fld (@CFG_PARAM)
			{
			$fld=&trim($fld);	
		
				if($fld=~ /^START\s*=\s*(\S+)/)
				{
				$JOB_INFO{"$region|$type|$job"}{'JOB_START'}=$1;
				}
			
				if($fld=~ /^END\s*=\s*(\S+)/)
				{
				$JOB_INFO{"$region|$type|$job"}{'JOB_END'}=$1;
				}
			
				if($fld=~ /^START_BUFFER\s*=\s*(\S+)/)
				{
				$JOB_INFO{"$region|$type|$job"}{'JOB_START_BUFFER'}=$1;
				}
				
				if($fld=~ /^STATUS_SLA\s*=\s*(\S+)/)
				{
				$JOB_INFO{"$region|$type|$job"}{'JOB_STATUS_SLA'}=$1;
				}
				if($fld=~ /^JOB_DEPEND\s*=\s*(\S+)/)
				{
				my $dep=$1;
					if($dep=~ /\|/)
					{
					$JOB_INFO{"$region|$type|$job"}{'JOB_DEPEND'}="$dep";
					}
					else
					{
					$JOB_INFO{"$region|$type|$job"}{'JOB_DEPEND'}="$region|$type|$dep";
					}
				}
				if($fld=~ /^JOB_DEPEND_RUN_TIME\s*=\s*(\S+)/)
				{
				$JOB_INFO{"$region|$type|$job"}{'JOB_DEPEND_RUN_TIME'}=$1;
				}
			}
		}
	}
close CFG;
}

#####################################################################

sub populate_info
{
	foreach my $job (keys %JOB_INFO)
	{
		if(defined $JOB_INFO{$job}{'JOB_START'} && defined $JOB_INFO{$job}{'JOB_END'} && defined $JOB_INFO{$job}{'JOB_START_BUFFER'})
		{
		$JOB_MONITOR{"$job|$JOB_INFO{$job}{'JOB_START'}|$JOB_INFO{$job}{'JOB_END'}|$JOB_INFO{$job}{'JOB_START_BUFFER'}"}='start';
		}
		if(defined $JOB_INFO{$job}{'JOB_STATUS_SLA'})
		{
			if(defined $SLA_INFO{$JOB_INFO{$job}{'JOB_STATUS_SLA'}})
			{
			my $prev_job=$SLA_INFO{$JOB_INFO{$job}{'JOB_STATUS_SLA'}};
			my $jobs="$prev_job~$job";
			$SLA_INFO{$JOB_INFO{$job}{'JOB_STATUS_SLA'}}=$jobs;
			}
			else
			{
			$SLA_INFO{$JOB_INFO{$job}{'JOB_STATUS_SLA'}}=$job;
			}
		}

		if(defined $JOB_INFO{$job}{'JOB_DEPEND'} && !defined $JOB_INFO{$job}{'JOB_DEPEND_RUN_TIME'} && !defined $JOB_INFO{$job}{'JOB_START_BUFFER'})
		{
		writeLog("err","Incorrect config, JOB_DEPEND_RUN_TIME=|START_BUFFER= Not defined for JOB \'$job\'");
		exit 1;
		}
		if(defined $JOB_INFO{$job}{'JOB_DEPEND'} && defined $JOB_INFO{$job}{'JOB_DEPEND_RUN_TIME'} && defined $JOB_INFO{$job}{'JOB_START_BUFFER'})
		{
		$JOB_DEPENDS_INFO{$job}="$JOB_INFO{$job}{'JOB_DEPEND'}~$JOB_INFO{$job}{'JOB_DEPEND_RUN_TIME'}~$JOB_INFO{$job}{'JOB_START_BUFFER'}";
		}
	}

### Sort SLA INFO
#print "BEFORE:\n";
#print Dumper(\%SLA_INFO);

	foreach my $key (keys %SLA_INFO)
	{
	my @VAL=split(/\~/,$SLA_INFO{$key});
	@VAL=sort @VAL;
	my $val=join("~",@VAL);
	$SLA_INFO{$key}=$val;
	}
#print "AFTER:\n";
#print Dumper(\%SLA_INFO);


	foreach my $job (keys %JOB_DEPENDS_INFO)
	{
	my ($key,$rest)=split(/~/,$job,2);
	my ($dep,$rest1)=split(/~/,$JOB_DEPENDS_INFO{$job},2);
	$JOB_DEPENDS_INFO_CHK{$key}=$dep;
	}

}

#####################################################################

sub addTime
{	
my ($timestring,$diff)=(shift,shift);
#print (" paassedtime $timestring $diff \n");
my @stmp= split('[\s]+',$timestring,2);
my @dt=split('/',$stmp[0],3);
my @ts=split(':',$stmp[1],3);
#print ($dt[2]);
#3/10/2013print ($ts[0]);
my $ts = DateTime->new(
        		year      => $dt[2], 
        		month     => $dt[1],
        		day     => $dt[0],
        		hour      => $ts[0],
        		minute    => $ts[1],
        		second    => $ts[2]);

my $duration = DateTime::Duration->new(minutes => $diff);
my $nTs = $ts->add($duration);
my $ntime=$nTs->strftime("%d/%m/%Y %H:%M:%S");
#print ("return time $ntime\n");
return $ntime;
}	

###############################################################################

sub get_job_status
{
my ($job)=@_;

my $res=`autorep -J $job | grep $job `;
		
$res=&trim($res);
writeLog("job_status","$res");	
	
	if($res ne "")
	{
		if($res=~ m|(.+?)\s+(\d{2}/\d{2}/\d{4})\s+(\d{2}:\d{2}(:\d{2})?)\s+(\d{2}/\d{2}/\d{4})\s+(\d{2}:\d{2}(:\d{2})?)\s+(\w{2})\s+|g)
		{
		my $job=$1;
		my $st_date=$2;
		my $st_time=$3;
		my $fin_date=$5;
		my $fin_time=$6;
		my $status=$8;

		$job=sprintf("%-35s",$job);
		$st_time='On Ice', $fin_time='On Ice'	if($status eq 'OI');
		$st_time='OnHold', $fin_time='OnHold'	if($status eq 'OH');
		return ($res,$status, $st_date, $st_time, $fin_date,$fin_time);
		}
		elsif($res =~ m|(.+?)\s+\-\-\-\-\-\s+(\d{2}/\d{2}/\d{4})\s+(\d{2}:\d{2}(:\d{2})?)\s+(\w{2})\s+|)
		{
		my $job=$1;
		my $fin_date=$2;
		my $fin_time=$3;
		my $status=$5;	
		my $st_date='xx/xx/xxxx';
		my $st_time='--:--:--';
		$job=sprintf("%-35s",$job);
		$st_time='On Ice', $fin_time='On Ice'	if($status eq 'OI');
		$st_time='OnHold', $fin_time='OnHold'	if($status eq 'OH');
		return ($res,$status, $st_date, $st_time, $fin_date,$fin_time);
		}
		elsif($res =~ m|(.+?)\s+(\d{2}/\d{2}/\d{4})\s+(\d{2}:\d{2}(:\d{2})?)\s+\-\-\-\-\-\s+(\w{2})\s+|)
		{
		my $job=$1;
		my $st_date=$2;
		my $st_time=$3;
		my $status=$5;
		my $fin_date='xx/xx/xxxx';
		my $fin_time='--:--:--';
		$job=sprintf("%-35s",$job);
		$st_time='On Ice', $fin_time='On Ice'	if($status eq 'OI');
		$st_time='OnHold', $fin_time='OnHold'	if($status eq 'OH');
		# print "$res,$status, $st_date, $st_time, $fin_date,$fin_time\n";
		return ($res,$status, $st_date, $st_time, $fin_date,$fin_time);
		}
		elsif($res =~ m|(.+?)\s+\-\-\-\-\-\s+\-\-\-\-\-\s+(\w{2})\s+|)
		{
		my $job=$1;
		my $status=$2;
		my $st_date='xx/xx/xxxx';
		my $st_time='--:--:--';
		my $fin_date='xx/xx/xxxx';
		my $fin_time='--:--:--';
		$job=sprintf("%-35s",$job);
		$st_time='On Ice', $fin_time='On Ice'	if($status eq 'OI');
		$st_time='OnHold', $fin_time='OnHold'	if($status eq 'OH');
		return ($res,$status, $st_date, $st_time, $fin_date,$fin_time);
		}
		else
		{
		print "NOT_PROCESSED: --$res--\n";
		exit(108);
		}

	}
return;
}
#####################################################################

sub check_attr
{
my ($key,$curr_ts)=@_;
my ($curr_date,$curr_time) =split('[\s]+',$curr_ts);
my @job_arr=split('\|',$key);

#print Dumper(\@job_arr);
my $job_key="$job_arr[0]|$job_arr[1]|$job_arr[2]";

my $run_dt=&get_format_date($rundate,'yyyymmdd','dd/mm/yyyy');
my $ideal_ts="$run_dt"." "."$job_arr[3]";
print "CURRENT TIME      :$curr_ts\n";
print "IDEAL_START       :$ideal_ts\n";

my $job_check_time=addTime($ideal_ts,$job_arr[5]);
print "IDEAL_START_BUFFER:$job_check_time\n"; 

#my $ideal_ts="$curr_date"." "."$job_arr[3]";

##print "--$td--\n";
my ($res,$status, $st_date, $st_time, $fin_date,$fin_time)=&get_job_status($job_arr[2]);
#print("$res,$status, $st_date, $st_time, $fin_date,$fin_time \n");
$st_date=&get_format_date($st_date,'mm/dd/yyyy','dd/mm/yyyy');
$fin_date=&get_format_date($fin_date,'mm/dd/yyyy','dd/mm/yyyy');

my $start_ts="$st_date"." "."$st_time";
my $end_ts="$fin_date $fin_time";


print "JOB START         :$start_ts\n";
print "JOB END           :$end_ts\n"; 

#print("$res,$status, $st_date, $st_time, $fin_date,$fin_time \n");

	
	if (defined $JOB_BAD_STATUS{$status})          ##if job is failed then send alert.
	{
	writeLog("wrn","Job Has Bad Status [$status] for key -> [$key|$status], job Rundate [$rundate], Mailing Rundate [$status_email_rundate]");
	&alert_job("-e","$key|$status~Job_Issue-bad_job_status","$job_arr[2] \[$job_arr[0]\]","<tr><td>$job_arr[0]</td><td>$job_arr[1]</td><td>$job_arr[2]</td><td>$res</td></tr>","Please Action on Job [ $job_arr[2] ], below is the Current Status:");
	my %ATTR=();			
	$ATTR{$key}{"JOB_OK"}="NO";
	return(\%ATTR);
	}
	elsif($status eq 'SU')
	{
	my $diff;
		if(defined $JOB_DEPENDS_INFO_CHK{$job_key})
		{
		writeLog("DEPENDENCY_VALIDATION","[$job_key] => [$JOB_DEPENDS_INFO_CHK{$job_key}]");

		$diff=tdiff($start_ts,$ideal_ts);
		}
		else
		{
		$diff=tdiff($curr_ts,$job_check_time);
		}


		if ($diff <= 0)
		{
		writeLog("JOB_RUN_Validation","Time Difference [$diff], (NEGATIVE|ZERO) value Means-> Job Run Time is MET and Status is $status");
		###Valiadtion
		my $job_ok=0;
			my $start_diff=tdiff($start_ts,$job_check_time);
			#my $end_diff=tdiff($job_check_time,$end_ts);
			my $end_diff=tdiff($ideal_ts,$end_ts);
			#print "~~~~~~~~~~~~CHECK_CONDITION_JOB_COMPLETION -> Sart_Diff [$start_diff > 0] AND End_Diff [$end_diff > 0]\n";
			
			#writeLog("JOB_RUN_validation","Time Difference [$end_diff], if (POSITIVE) value -> Job is COMPLETED and Status is SU");

			writeLog("job_RUN_info","CHECK_CONDITION_JOB_COMPLETION -> End_Diff [$end_diff] and Strat Diff [$start_diff]");
			
			#if($start_diff > 0 && $end_diff > 0)
			if($end_diff > 0)
			{
			writeLog("dum","");
			writeLog("info","Job has Started and Completed for job Rundate [$rundate], Mailing Rundate [$status_email_rundate]");
			$job_ok=1;
			}
			if($job_ok)
			{
			my %ATTR=();
			writeLog("dum","");
			&writeLog("Info","job [$job_arr[2]] Start time is MET -> STATUS [$status], job Rundate [$rundate], Mailing Rundate [$status_email_rundate]");	

			$ATTR{$key}{"JOB_OK"}="YES";
			$ATTR{$key}{"STATUS"}=$status;
			$ATTR{$key}{"START_DATE"}=$st_date;
			$ATTR{$key}{"START_TIME"}=$st_time;
			$ATTR{$key}{"END_DATE"}=$fin_date;
			$ATTR{$key}{"END_TIME"}=$fin_time;
			return (\%ATTR);
			}
			else
			{
			writeLog("dum","");
			writeLog("wrn","Job [$job_arr[2]] Didn't kicked off for key -> [$key], job Rundate [$rundate], Mailing Rundate [$status_email_rundate]");
			&alert_job("-e","$key|$status~Job_Issue-did_not_kicked_off","$job_arr[2] \[$job_arr[0]\]","<tr><td>$job_arr[0]</td><td>$job_arr[1]</td><td>$job_arr[2]</td><td>$res</td></tr>","Please Action on Job [$job_arr[2]] which didn't Kicked off for Rundate $rundate");
			my %ATTR=();			
			$ATTR{$key}{"JOB_OK"}="NO";
			return(\%ATTR);
			}
		}
		else
		{
		writeLog("JOB_RUN_validation","Time Difference [$diff], if (POSITIVE) value Means-> Job Run Time is NOT MET but Status is $status");
		writeLog("dum","");
		my %ATTR=();
		&writeLog("Info","job [$job_arr[2]] Start time is NOT met for key -> $key], job Rundate [$rundate], Mailing Rundate [$status_email_rundate] ");	
		$ATTR{$key}{"JOB_OK"}="NO";
		return(\%ATTR);
		}
	}
	elsif($status eq 'RU')
	{
	my $diff;
		#print "KKKKKKKKKKKKKKKKKK: $job_key\n";
		#print Dumper(\%JOB_DEPENDS_INFO_CHK);
		if(defined $JOB_DEPENDS_INFO_CHK{$job_key})
		{
		writeLog("DEPENDENCY_VALIDATION","[$job_key] => [$JOB_DEPENDS_INFO_CHK{$job_key}]");
		print "$job_check_time   $start_ts\n";
		$diff=tdiff($job_check_time,$start_ts);
		}
		else
		{
		$diff=tdiff($curr_ts,$job_check_time);
		}

	my $job_ok=0;
	writeLog("job_RUN_info","CHECK_CONDITION_JOB_RUNNING -> Strat Diff [$diff]");

		if ($diff <= 0)
		{
		writeLog("dum","");
		writeLog("info","Job has Started and in Running State for job Rundate [$rundate], Mailing Rundate [$status_email_rundate]");
		$job_ok=1;
		}
		if($job_ok)
		{
		#print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n";
		writeLog("JOB_RUN_validation","Time Difference [$diff], (NEGATIVE|ZERO) value Means-> Job Run Time is MET and Status is $status");
		my %ATTR=();

		&writeLog("Info","job [$job_arr[2]] Start time is MET -> STATUS [$status], job Rundate [$rundate], Mailing Rundate [$status_email_rundate]");	
		$ATTR{$key}{"JOB_OK"}="YES";
		$ATTR{$key}{"STATUS"}=$status;
		$ATTR{$key}{"START_DATE"}=$st_date;
		$ATTR{$key}{"START_TIME"}=$st_time;
		$ATTR{$key}{"END_DATE"}=$fin_date;
		$ATTR{$key}{"END_TIME"}=$fin_time;
		return (\%ATTR);
		}
		else
		{
		writeLog("JOB_RUN_validation","Time Difference [$diff], (POSITIVE) value Means-> Job Run Time is NOT MET but Status is $status");
		writeLog("dum","");
		my %ATTR=();
		&writeLog("Info","job [$job_arr[2]] Start time is NOT met for key -> $key], job Rundate [$rundate], Mailing Rundate [$status_email_rundate] ");	
		$ATTR{$key}{"JOB_OK"}="NO";
		return(\%ATTR);
		}
	}
	else
	{
	print "\n";
	print "SOME ISSUE::\nNeed to Check manually\n";
	print("$res,$status, $st_date, $st_time, $fin_date,$fin_time \n");
	print "\n";
	}
}

#####################################################################


sub alert_job
{
my ($mode,$key,$job,$body, $job_dt)=@_;
my $wfh;
my $already_sent_flg=0;

	if($mode eq "-e")
	{
	#writeLog("DUM","");
	#writeLog("info","$body");
	$already_sent_flg=&check_log($MAIL_LOG_FILE,$key);
	
		unless($already_sent_flg)
		{
		my $mbh=IO::File->new("$mail_body_file","w")|| die "Can't open the File [$mail_body_file] in Write mode : $!\n";;
		$mbh->print("$job_dt<br><br>\n");
		$mbh->print("<TABLE border='2' cellpadding='2' style='border-collapse: collapse' bordercolor='#111111' width='95%' ><TR bgcolor='#66ccff' ><td>Region</td><td>Type Process</td><td>Job Name</td><td>Status</td></TR>\n");
		
		$mbh->print ("$body\n");
		$mbh->print("</table>\n");
		
		my $res=system("$ENV{SCRIPTSROOT}/util/runperl.csh /apps/rvg/bin/util/send_mail_util.pl -f job_alert_operate -c $mail_config -d $status_email_rundate -s \'$job\'");
		
			if($res ==0)
			{
			writeLog("info","Mail Sent Successfully");
			append_log_file('append',$key);
			}
			else
			{
			writeLog("err","Failed to Send Mail");
			exit 1;
			}
		}
	}
}
#####################################################################

sub check_log
{
my ($MAIL_LOG_FILE,$search)=@_;

$search=~ s/\|/~/g;
#writeLog("dum","");
	if (-f $MAIL_LOG_FILE)
	{
	my $res=system("grep $search $MAIL_LOG_FILE > /dev/null 2>&1");

		if($res == 0)
		{
		#writeLog("info","Found Search Info [$search] in Mail Log File [$MAIL_LOG_FILE]");
		return 1;
		}
		else
		{
		#writeLog("info","NOT Found Search Info [$search] in Mail Log File [$MAIL_LOG_FILE]");
		return 0;
		}
	}
	else
	{
	return 0;
	}
}
#####################################################################

sub append_log_file
{
my ($ah,$key)=@_;
#my $TimeStamp=`date +[%Y-%m-%d' '%l:%M:%S' '%p]`;
my $TimeStamp=`date +[%Y-%m-%d' '%H:%M:%S]`;
chomp $TimeStamp;
 
	if(-f $MAIL_LOG_FILE)
	{
	#	if($ah eq 'append')
	#	{
		#print "---$ah---\n";
	open(FF,">>$MAIL_LOG_FILE") || die "Can't open the the File in Append Mode [$MAIL_LOG_FILE] :$!\n";
	#	}
	}
	else
	{
	#print "----CREATE $ah---\n";
	open(FF,">$MAIL_LOG_FILE") || die "Can't open the the File in Create Mode [$MAIL_LOG_FILE] :$!\n";
	}

$key=~ s/\|/~/g;
print FF "$TimeStamp~$key\n";
sleep 1;
close FF;
}

#####################################################################

sub tdiff($$)
{
my ($t1,$t2)=(shift,shift);

#print "T1:$t1 --T2:$t2\n";
my @idarr=split('[\s]+',$t1,2);
my @fdarr=split('[\s]+',$t2,2);
my @s1darr=split('/',$idarr[0],3);
my $mon1=$s1darr[1] -1;
$mon1=sprintf("%02d",$mon1);
$s1darr[1]=$mon1;
my @s2darr=split('/',$fdarr[0],3);
my $mon2=$s2darr[1] -1;
$mon2=sprintf("%02d",$mon2);
$s2darr[1]=$mon2;
my @s1tarr=split(':',$idarr[1],3);
my @s2tarr=split(':',$fdarr[1],3);

#print "S1:$s1tarr[2],$s1tarr[1],$s1tarr[0],$s1darr[0],$s1darr[1],$s1darr[2]\n";
#print "S2:$s2tarr[2],$s2tarr[1],$s2tarr[0],$s2darr[0],$s2darr[1],$s2darr[2]\n";
my $et1=timelocal($s1tarr[2],$s1tarr[1],$s1tarr[0],$s1darr[0],$s1darr[1],$s1darr[2]);
my $et2=timelocal($s2tarr[2],$s2tarr[1],$s2tarr[0],$s2darr[0],$s2darr[1],$s2darr[2]);

my $diff_time=($et2 - $et1);
my $diff=$diff_time/60;
#print "T2 - T1 /60\n";
#print "DIFF:($et2 - $et1)/60 : $diff\n";
return $diff;
}

#####################################################################

sub usage
{
my $HELP=<<EOD;

Usage : $0
    -c : Config File Name                       (Optional)
    -d : Check Job Rundate           <yyyymmdd> (Optional)
    -m : Status Email Effective Date <yyyymmdd> (Optional)
    -r : Region <ASIA|EMEA|NA|etc..             (Optional)
    -s : Search <Staging|DW_Batch_Status|etc..>	(Optional)
    -h : Help

EOD
	
print $HELP;
}

=head
Sample config
#REGION=REG_name ~ TYPE=kind_of_task JOB=JobName ~ START=HH:MM:SS ~ END=HH:MM:SS ~ START_BUFFER=MIN ~ STATUS_SLA=HH:MM:SS ~ JOB_DEPEND=JOBNAME ~ JOB_DEPEND_RUN_TIME=MIN

**NUMER_OF_ITERATION=500
**SLEEP_SEC=60

**MAIL_CFG=/apps/rvg/cfg/send_mail_monitor_job.xml
**AMBER=5


#Region__type=body_file (mapping)
--ASIA__Staging_run1=/cfg/ASIA__Staging_body_file.htm
--ASIA__DW_Batch_Status_run1=/cfg/ASIA__DW_Batch_Status_body_file.htm

############################### ASIA RUN1 ###########################

#Asia Dw batch Status mail
REGION=ASIA ~ TYPE=DW_Batch_Status_run1 ~ JOB=cioa_dw_work_flow_load_strategy_rating ~ START=01:00:00 ~ END=02:15:01 ~ START_BUFFER=65 ~ STATUS_SLA=22:30:00
REGION=ASIA ~ TYPE=DW_Batch_Status_run1 ~ JOB=p02.cio.tfa.ptfa.vh01.spar.update_db.ap.box ~ START=19:40:00 ~ END=22:30:00 ~ START_BUFFER=100 ~ STATUS_SLA=22:30:00
REGION=ASIA ~ TYPE=DW_Batch_Status_run1 ~ JOB=cioa_dw_workflow_box ~ START=20:00:00 ~ JOB_DEPEND=p02.cio.tfa.ptfa.vh01.spar.update_db.ap.box ~ JOB_DEPEND_RUN_TIME=15 ~ START_BUFFER=120 ~  STATUS_SLA=22:30:00
REGION=ASIA ~ TYPE=DW_Batch_Status_run1 ~ JOB=cio_dw_incr_stg_dim_fact_adj_sas_box ~ JOB_DEPEND=cioa_dw_workflow_box ~ JOB_DEPEND_RUN_TIME=150 ~ START_BUFFER=150 ~ STATUS_SLA=22:30:00

=cut