#!/usr/bin/perl
# use strict;
use Getopt::Std;
use Getopt::Long;
use FileHandle;
use Fcntl; 

$filemode = '0744';
$filemode = oct($filemode);
getopts('ROICTpm:e:c:q:o:s:h');

if ($opt_h) {
    print "Usage: pbsubmit -c \"command\" [-p] [-q queue] ";
    print "[-s shell] [-e environment] [-o qsub options] [-p] [-C] [-I]\n";
    print "[-Z skeleton]\n\n";
    print "Options: \n";
    print "[-h] Prints out this message.\n";
    print "[-e environment] selects a PBS environment settiing. (not implemented)\n";
    print "[-m] emails you when the job begins and ends.\n";
    print "[-o qsub options] passes options to the call to qsub.\n";
    print "[-p] begins the script in your present working directory.\n";
    print "[-q queue] selects a queue.\n";
    print "[-s shell] selects a shell.\n";
    print "[-C] writes a comment in the script for convenience.\n";
    print "[-I] uses qsub's interactive mode. \n";
    print "[-O] Overrides sleep-mode (which I use for letting load averages creep up.\n";
    print "[-R] submits a second job as a failsafe.\n";
    print "[-T] Test mode. Only writes the script, but does not submit it for queuing.\n";  

    print "[-Z skeleton arguments ] (In Beta) Generates a sequence of jobs from a skeleton file \n"; 
    print "[-P] (Not yet implemented.) Writes the settings for a LAM-MPI job.\n"; 
    exit ;
}
$opt_Z = 0;

&GetOptions("Z=s" => \$opt_Z);

# delay to let load averages settle
if (!$opt_O) {
    autoflush STDOUT 1;
    $curtime = time; 

    open (TIMEFILE, "/usr/spool/PBS/pbsubmit.time")||
	die "could not open pbsubmit.time\n";
    unless ( flock(TIMEFILE , LOCK_SH | LOCK_NB)){ 
	local $| =1 ; 
	print "Waiting for lock on timing file.\n"; 
	flock(TIMEFILE, LOCK_SH); or die "Can't lock timing file.\n";
    }
    $opt_O = <TIMEFILE>;
    $filetime = $opt_O;

#    close(TIMEFILE);
#    open (TIMEFILE, ">/usr/spool/PBS/pbsubmit.time")||
#	die "could not open pbsubmit.time\n";
    
#    print TIMEFILE "$curtime\n";


    if ($curtime < $filetime) {
	print STDERR "Delay set in to prevent overshoots. Use &.\n";
	print STDERR "Dum dee dum.. ";	
	while ($curtime < $filetime) {
	    sleep 4;
	    print STDERR ".zz.";
	    $curtime = time; 
	}
	print STDERR "\n*Yawn*... back to work for me.\n";
    }
    $filetime = $curtime +60;
    close(TIMEFILE);
    sysopen(TIMEFILE, /usr/spool/PBS/pbsubmit.time, O_WRONLY | O_CREAT) 
	or die "could not open pbsubmit.time\n";
    flock(TIMEFILE,LOCK_EX) 
	or die "could not lock pbsubmit.time\n";
    truncate(TIMEFILE,0) 
	or die "could not truncate pbsubmit.time\n";
#    open (TIMEFILE, ">/usr/spool/PBS/pbsubmit.time")||
#	
    print TIMEFILE "$filetime\n";
    close(TIMEFILE);
}
if ($opt_m) {
    $opt_m = "-m abe -M " . $opt_m;
}

$new_job = &get_jobfile_number(); 

print STDERR "Openning $new_job\n";
 die "$new_job already exists. Time to purge .pbsrc.\n" if -e $new_job;

if (!$opt_s) {
    $opt_s = "/bin/sh";
}
if (!$opt_c) {
    die "No command specified. Exiting.\n";
}
if ($opt_q){
    $opt_q = "-q " . $opt_q;
}
if ($opt_C) {
    print "-C invoked. Please give this job a name or comment. (One line.) \n";
    $opt_C = <STDIN>;
    print JOBFILE "#COMMENT: $opt_C \n";
}


$JobFile = new FileHandle "> $new_job"; 

if (defined $JobFile) { 
    &print_jobfile_header($JobFile);
} else {
    die "Could not open $new_job to print the header.\n";
}
$homedir = $ENV{HOME}; 
$job_status = $new_job;
$job_status .= ".status" ;

&handle_jobfile_options($JobFile) ;
&print_jobfile_command($JobFile,$homedir,$job_status); 
&print_jobfile_footer($JobFile, $homedir, $job_status); 
$JobFile->close;

chmod (0755, $new_job);
if ($opt_I) {
    $opt_o = "-I " . $opt_o;
}

$frobfile = new FileHandle "> $job_status";

if (defined $frobfile ) { 
    $frobfile->autoflush(1); 
    &print_statusfile_header($frobfile); 
} else {die "Could not open status file $job_status.\n";}

print "qsub $opt_o $opt_q $homedir/.pbsrc/$new_job \n";
#test mode...
if ($opt_T ) { 
    print $frobfile "# Script was not queued in. User invoked -T option.\n";
} else { 
    
    open(QSUBPROC, "qsub $opt_o $opt_m $opt_q $homedir/.pbsrc/$new_job 2>&1 |");
    select(QSUBPROC);
    $| =1;
    select(STDOUT);
    
#get a good qsub parse!!!
    print $frobfile $opt_C;
    while (<QSUBPROC>) {
	print $frobfile "Queued $_";
	print STDOUT $_; 
	s/[^\d]//g;
	
	$qsub_output = $_;
	
    }
    close QSUBPROC;
    if ($opt_R) {
	$opt_m =~ s/-m abe/-m be/;
# retries.
#    print "qsub $opt_o $opt_q -W depend=afternotok:$qsub_output";
#    print	" $homedir/.pbsrc/$new_job\n ";
	open(QSUBPROC2,
	     "qsub $opt_o $opt_q $opt_m -W depend=afternotok:$qsub_output $homedir/.pbsrc/$new_job  |");
	select(QSUBPROC2);
	$| =1;
	select(STDOUT);
	
#    print $frobfile $opt_C;
	
	while (<QSUBPROC2>) {
	    print $frobfile "queued $_";
	    
	    print STDOUT $_; 
	}
	close QSUBPROC2;
	
    }
}
$frobfile->close;
$frobfile->open(">>$job_status") ||
    die "Could not reopen $job_status.\n";
print $frobfile "#and the job is...\n\n\n";
$frobfile->close;

######################
# handle_jobfile_options 
#
######################
sub handle_jobfile_options  {
    my ($filehandle) = @_;
    if (defined $filehandle ) { 
	if ($opt_e) {
	    print $filehandle "# setting directives for PBS: \n";
	    print $filehandle "#PBS -l $opt_e \n";
	}
	if ($opt_p) {
	    $opt_p = $ENV{pwd};
	    print $filehandle "# implementing option -p\n";
	    print $filehandle "cd $opt_p \n" ;
	}

    }
} 



########################
# get_jobfile_number 
#
# gets a new jobfile number.
########################
sub get_jobfile_number { 

    my ($homedir, $new_job, $last_job, @jobs_dir) ; 

    $homedir = $ENV{"HOME"};
    unless(opendir(JOBSDIR, "$homedir/.pbsrc")) {
	print STDERR "no .pbsrc directory. Making one. \n";
	mkdir("$homedir/.pbsrc",0744)  ||
	    die "Could not make .pbsrc. Something is wrong.\n\n" ;
	
    }
    @jobs_dir = grep /pbsjob_[1234567890]+$/, readdir JOBSDIR;
#    print @jobs_dir;
    closedir JOBSDIR;
    chdir "$homedir/.pbsrc" || die "Could not chdir to .pbsrc";
    
    if ($#jobs_dir > 1000) { 
	print "It may be time to purge .pbsrc. \n";
    }
    $new_job=1;
    foreach $last_job (@jobs_dir) { 
	$last_job =~ s/^pbsjob_// ;
	
	if ($last_job > $new_job) {
	    $new_job = $last_job ; 
	}
    }
    $new_job ++; 
    $new_job = "pbsjob_$new_job"; 
    return $new_job;

}

###################

# print_jobfile_command : prints 
# command part of a job file.
#
###################

sub print_jobfile_command { 

    my ($filehandle,$homedir, $job_status) = @_;
    if (defined $filehandle ) { 

	print $filehandle "#These lines added for accounting while job runs:\n";
	print $filehandle "hostname >>   $homedir/.pbsrc/$job_status\n";
	print $filehandle "echo running `date` >> $homedir/.pbsrc/$job_status\n";
	
	
	print $filehandle "# The command submitted was:\n";
	print $filehandle "# CMD: $opt_c\n";
	if ($opt_P) { print $filehandle '#'; } 
	print $filehandle "$opt_c\n";
	if ($opt_P) { 
# set for LAM-MPI. Other implementations will require other things
	    print $filehandle "# The -P flag is set for parallel execution.\n";
	    print $filehandle "# $opt_P nodes are requested, hence the line below:.\n";
	    print $filehandle "#PBS -l nodes=$opt_P \n";
	    print $filehandle "# Now a few setup commands. The job will run on these process
ors: \n";
	    print $filehandle "echo machines used: >>  $homedir/.pbsrc/$job_status\n";
	    print $filehandle "echo `cat `\$PBS_NODEFILE` >>  $homedir/.pbsrc/$job_status\n"
		;
	    print $filehandle "NPROCS=`wc -l < \$PBS_NODEFILE`";
	    if ($opt_P eq 'mpich' ) {
		print $filehandle "# the chosen implementation is MPICH, hence this:\n";
		print $filehandle "mpirun -v -machinefile $PBS_NODEFILE -np $NPROCS $opt_c\n
";
	    } elsif ($opt_P eq 'lam' ) { 
		print $filehandle "# the chosen implementation is LAM, hence this:\n";
# command line needs a double dash...
		print $filehandle "lamboot -v \$PBS_NODEFILE\n"
		$opt_c =~ s/\b/ -- /; 
		print $filehandle "mpirun -v C $opt_c\n";
		print $filehandle "wipe -v \$PBS_NODEFILE\n"
	    } 
	    
#    $mpi_nodes = `cat $PBS_NODEFILE`; 
#to do: reparse opt_c for mpirun.
# add a version of lamboot.
# add mpirun. add lamclean
	    
	}
    }
} 


###################
# print_jobfile_header : prints the beginning 
# (and unchanging) part of a job file.
#
###################

sub print_jobfile_header { 

    my ($filehandle) = @_;

    if (defined $filehandle ) { 

	print $filehandle "#!$opt_s\n";
	print $filehandle "# This file was generated automatically.\n"; 
	print $filehandle "# on DATE ";
	print $filehandle scalar localtime;
	print $filehandle "\n";
    }
}

###################
# print_jobfile_footer : prints the final 
# (and unchanging) part of a job file.
#
###################

sub print_jobfile_footer { 

    my ($filehandle,$homedir,$job_status) = @_;

    if (defined $filehandle ) { 
	print $filehandle "#This line added for error number accounting: \n";
	print $filehandle "PBS_ERR=\$?\n";
	print $filehandle "# These lines added for accounting:\n";
	print $filehandle "echo done `date` status \${PBS_ERR} >> $homedir/.pbsrc/$job_status\n";
	
	print $filehandle "exit \$PBS_ERR";

    }
}
###################
# print_statusfile_header : prints the beginning 
# (and unchanging) part of a status file.
#
###################

sub print_statusfile_header { 

    my ($filehandle) = @_;

    if (defined $filehandle ) { 

	print $filehandle "# This file was generated automatically.\n"; 
	print $filehandle "#\n";
	print $filehandle "# The command submitted was:\n";
	print $filehandle "$opt_c\n";
	print $filehandle "#qsub command line options used were: \n";
	print $filehandle "$opt_o\n";
	print $filehandle "#the shell selected was $opt_s\n";
	
	print $filehandle "#the queue: $opt_q\n";
	print $filehandle "#the qsub command output: \n";
	
    }
}