| 1 |
#!/usr/bin/perl |
| 2 |
|
| 3 |
# Here's how to run this script (from scratch) |
| 4 |
# ssh jsoc@j0 |
| 5 |
# cd /home/jsoc/exports |
| 6 |
# rm keep_running |
| 7 |
# exportmanage.pl -jsocdev procser=jsoc.export_procs & |
| 8 |
# To start the web version for jsoc.stanford.edu repeat the above but do: |
| 9 |
# rm keep_running_web |
| 10 |
# exportmanage.pl -jsocweb procser=jsoc.export_procs& |
| 11 |
# At some point, I'll add the production version of the script, which |
| 12 |
# would then be run as "exportmanage.pl -jsocpro &" |
| 13 |
# To start the debug test version for jsoc2 do: |
| 14 |
# rm keep_running_test |
| 15 |
# exportmanage.pl -jsoctest procser=jsoc.export_procs & |
| 16 |
|
| 17 |
# To test the entire export workflow: |
| 18 |
# 1. Run export manage like so: |
| 19 |
# su <USER> |
| 20 |
# where <USER> is the user whose $PATH contains the paths to the modules/scripts that have new code to be tested. |
| 21 |
# The manager program makes shell scripts that call programs and scripts by calling the program/script base file |
| 22 |
# name (not the full path to the program/script). So the actual program/script that runs will be the one |
| 23 |
# that the shell resolves using the $PATH variable. For example, if the user arta makes |
| 24 |
# changes to the jsoc_export_as_fits in arta's home directory, and arta's $PATH contains a pointer to the binaries |
| 25 |
# in arta's home directory, then arta should run 'su arta' before continuing. IMPORTANT - <USER> will need |
| 26 |
# permissions to write into /home/jsoc/exports/tmp and /home/jsoc/exports/logs. Generally <USER> will be a |
| 27 |
# member of the group jsoc, so this requirement will be met. |
| 28 |
# cd /home/jsoc/exports |
| 29 |
# /home/jsoc/cvs/Development/JSOC/proj/util/scripts/exportmanage.pl -root <ROOT> -dbuser <DBUSER> -dbhost <DBHOST> -manager <MANAGER> -runflag <RFLAG> & |
| 30 |
# where <ROOT> is the CVS code tree root containing <MANAGER> |
| 31 |
# and <DBUSER> is the PG user who the manager connects as (defaults to "production"). IMPORTANT - the manager will |
| 32 |
# write records to tables that require elevated permissions. Most likely, you'll need to connect to the database |
| 33 |
# as user production to do this. This means that you'll need to place the password for user production in |
| 34 |
# your .pgpass file. |
| 35 |
# and <DBHOST> is the host of the database server (defaults to "hmidb"). For internal exports, should be "hmidb", for exports from public db, should be "hmidb2" |
| 36 |
# and <MANAGER> is the name of the manager program (defaults to "jsoc_export_manage"). IMPORTANT - you should include |
| 37 |
# a "-t" flag. This will cause the manager program to run in test mode, which means that it will process records |
| 38 |
# in jsoc.export_new that contain the special test status of 12 (instead of the regular status of 2). |
| 39 |
# and <RFLAG> is the file flag that keeps this script running in a loop (defaults to keep_running in cdir) |
| 40 |
# |
| 41 |
# Example : /home/jsoc/cvs/Development/JSOC/proj/util/scripts/exportmanage.pl -root /home/arta/cvs/JSOC -dbuser production -dbhost hmidb2 -manager "jsoc_export_manage -t procser=jsoc.export_procs" -runflag keepruntest.txt -logflag Test & |
| 42 |
# |
| 43 |
# 2. Point a browser at http://jsoc.stanford.edu/ajax/exportdatatest.html and export something. |
| 44 |
|
| 45 |
use strict; |
| 46 |
use warnings; |
| 47 |
|
| 48 |
use FileHandle; |
| 49 |
use Fcntl ':flock'; |
| 50 |
use FindBin qw($RealBin); |
| 51 |
use lib "$RealBin/../../../localization"; |
| 52 |
use drmsparams; |
| 53 |
|
| 54 |
use constant kExportDir => "/home/jsoc/exports"; |
| 55 |
# use constant kMailList => "arta\@sun.stanford.edu jeneen\@sun.stanford.edu phil\@sun.stanford.edu"; |
| 56 |
use constant kMailList => "arta\@sun.stanford.edu"; |
| 57 |
use constant kMailMessage1 => "exportmanage.pl could not start jsoc_export_manage. This is a critical failure.\nYou should probably contact Art, who was also notified and should respond shortly.\n"; |
| 58 |
use constant kMailMessage2 => "jsoc_export_manage died in response to an unhandled signal (e.g., a segfault).\n"; |
| 59 |
use constant kMailMessage3 => "Could not open log export-daemon log file for writing.\nThis is not a critical failure, but we should\nfix this so that we can track-down future export problems more easily.\nContact Art.\n"; |
| 60 |
|
| 61 |
use constant kMsgType1 => "msgtype1"; |
| 62 |
use constant kMsgType2 => "msgtype2"; |
| 63 |
use constant kMsgType3 => "msgtype3"; |
| 64 |
|
| 65 |
use constant kMsgQInterval => 600; # 10 minutes, at least, between message inserts |
| 66 |
use constant kMsgQSendInterval => 120; # 2 minutes between mailing of messages |
| 67 |
|
| 68 |
use constant kLogFlagInt => "Int"; |
| 69 |
use constant kLogFlagExt => "Ext"; |
| 70 |
|
| 71 |
my($config) = new drmsparams; |
| 72 |
|
| 73 |
my($kINTERNALFLAG) = "/home/jsoc/exports/keep_running"; |
| 74 |
my($kWEBFLAG) = "/home/jsoc/exports/keep_running_web"; |
| 75 |
my($kTESTFLAG) = "/home/jsoc/exports/keep_running_test"; |
| 76 |
# |
| 77 |
my($kJSOCDEV_ROOT) = "/home/jsoc/cvs/Development/JSOC"; |
| 78 |
my($kJSOCDEV_DBUSER) = "production"; |
| 79 |
my($kJSOCDEV_DBNAME) = $config->get('DBNAME'); |
| 80 |
my($kJSOCDEV_DBHOST) = $config->get('SERVER'); |
| 81 |
my($kJSOCDEV_MANAGE) = "jsoc_export_manage"; |
| 82 |
use constant kProcInfoSeriesDev => "jsoc.export_procs"; |
| 83 |
# |
| 84 |
my($kJSOCPRO_ROOT) = "/home/jsoc/cvs/JSOC"; |
| 85 |
my($kJSOCPRO_DBUSER) = "production"; |
| 86 |
my($kJSOCPRO_DBNAME) = $config->get('DBNAME'); |
| 87 |
my($kJSOCPRO_DBHOST) = $config->get('SERVER'); |
| 88 |
my($kJSOCPRO_MANAGE) = "jsoc_export_manage"; |
| 89 |
use constant kProcInfoSeriesPro => "jsoc.export_procs"; |
| 90 |
# |
| 91 |
my($kJSOCWEB_ROOT) = "/home/jsoc/cvs/Development/JSOC"; |
| 92 |
my($kJSOCWEB_DBUSER) = "production"; |
| 93 |
my($kJSOCWEB_DBNAME) = "jsoc"; |
| 94 |
my($kJSOCWEB_DBHOST) = "hmidb2"; |
| 95 |
my($kJSOCWEB_MANAGE) = "jsoc_export_manage"; |
| 96 |
use constant kProcInfoSeriesWeb => "jsoc.export_procs"; |
| 97 |
# |
| 98 |
my($kJSOCTEST_ROOT) = "/home/jsoc/cvs/Development/JSOC"; |
| 99 |
my($kJSOCTEST_DBUSER) = "phil"; |
| 100 |
my($kJSOCTEST_DBNAME) = $config->get('DBNAME'); |
| 101 |
my($kJSOCTEST_DBHOST) = $config->get('SERVER'); |
| 102 |
my($kJSOCTEST_MANAGE) = "jsoc_export_manage_test"; |
| 103 |
use constant kProcInfoSeriesTst => "jsoc.export_procs"; |
| 104 |
|
| 105 |
my($runningflag) = $kINTERNALFLAG; |
| 106 |
my($arg); |
| 107 |
my($root); |
| 108 |
my($dbhost) = $config->get('SERVER'); |
| 109 |
my($dbname) = $config->get('DBNAME'); |
| 110 |
my($dbuser) = "production"; |
| 111 |
my($binpath); |
| 112 |
my($manage) = "jsoc_export_manage"; |
| 113 |
my($logfile); |
| 114 |
my($daemonlog); |
| 115 |
my($lckfh); |
| 116 |
my($msg); |
| 117 |
my($logflag); |
| 118 |
my($procser); |
| 119 |
|
| 120 |
while ($arg = shift(@ARGV)) |
| 121 |
{ |
| 122 |
if ($arg eq "-root") |
| 123 |
{ |
| 124 |
$root = shift(@ARGV); |
| 125 |
$binpath = "$root/bin"; |
| 126 |
} |
| 127 |
elsif ($arg eq "-dbhost") |
| 128 |
{ |
| 129 |
$dbhost = shift(@ARGV); |
| 130 |
} |
| 131 |
elsif ($arg eq "-dbuser") |
| 132 |
{ |
| 133 |
$dbuser = shift(@ARGV); |
| 134 |
} |
| 135 |
elsif ($arg eq "-dbname") |
| 136 |
{ |
| 137 |
$dbname = shift(@ARGV); |
| 138 |
} |
| 139 |
elsif ($arg eq "-manager") |
| 140 |
{ |
| 141 |
$manage = shift(@ARGV); |
| 142 |
} |
| 143 |
elsif ($arg eq "-runflag") |
| 144 |
{ |
| 145 |
$runningflag = shift(@ARGV); |
| 146 |
} |
| 147 |
elsif ($arg eq "-logflag") |
| 148 |
{ |
| 149 |
$logflag = shift(@ARGV); |
| 150 |
} |
| 151 |
elsif ($arg eq "-procser") |
| 152 |
{ |
| 153 |
$procser = shift(@ARGV); |
| 154 |
} |
| 155 |
elsif ($arg eq "-jsocdev") |
| 156 |
{ |
| 157 |
$root = $kJSOCDEV_ROOT; |
| 158 |
$binpath = "$root/bin"; |
| 159 |
$dbuser = $kJSOCDEV_DBUSER; |
| 160 |
$dbname = $kJSOCDEV_DBNAME; |
| 161 |
$dbhost = $kJSOCDEV_DBHOST; |
| 162 |
$manage = $kJSOCDEV_MANAGE; |
| 163 |
$runningflag = $kINTERNALFLAG; |
| 164 |
$logflag = kLogFlagInt; |
| 165 |
$procser = &kProcInfoSeriesDev; |
| 166 |
} |
| 167 |
elsif ($arg eq "-jsocpro") |
| 168 |
{ |
| 169 |
$root = $kJSOCPRO_ROOT; |
| 170 |
$binpath = "$root/bin"; |
| 171 |
$dbuser = $kJSOCPRO_DBUSER; |
| 172 |
$dbname = $kJSOCPRO_DBNAME; |
| 173 |
$dbhost = $kJSOCPRO_DBHOST; |
| 174 |
$manage = $kJSOCPRO_MANAGE; |
| 175 |
$runningflag = $kINTERNALFLAG; |
| 176 |
$logflag = kLogFlagInt; |
| 177 |
$procser = &kProcInfoSeriesPro; |
| 178 |
} |
| 179 |
elsif ($arg eq "-jsocweb") |
| 180 |
{ |
| 181 |
$root = $kJSOCWEB_ROOT; |
| 182 |
$binpath = "$root/bin"; |
| 183 |
$dbuser = $kJSOCWEB_DBUSER; |
| 184 |
$dbname = $kJSOCWEB_DBNAME; |
| 185 |
$dbhost = $kJSOCWEB_DBHOST; |
| 186 |
$manage = $kJSOCWEB_MANAGE; |
| 187 |
$runningflag = $kWEBFLAG; |
| 188 |
$logflag = kLogFlagExt; |
| 189 |
$procser = &kProcInfoSeriesWeb; |
| 190 |
} |
| 191 |
elsif ($arg eq "-jsoctest") |
| 192 |
{ |
| 193 |
$root = $kJSOCTEST_ROOT; |
| 194 |
$binpath = "$root/bin"; |
| 195 |
$dbuser = $kJSOCTEST_DBUSER; |
| 196 |
$dbname = $kJSOCTEST_DBNAME; |
| 197 |
$dbhost = $kJSOCTEST_DBHOST; |
| 198 |
$manage = $kJSOCTEST_MANAGE; |
| 199 |
$runningflag = $kTESTFLAG; |
| 200 |
$logflag = "Test"; |
| 201 |
$procser = &kProcInfoSeriesTst; |
| 202 |
} |
| 203 |
} |
| 204 |
|
| 205 |
# Only run on j0.Stanford.EDU |
| 206 |
# if ($ENV{HOSTNAME} ne "j0.Stanford.EDU") { |
| 207 |
# die "I will only run on j0.Stanford.EDU\n"; |
| 208 |
#} |
| 209 |
|
| 210 |
# Don't run if somebody is already managing the export |
| 211 |
$lckfh = FileHandle->new(">$runningflag.lck"); |
| 212 |
unless (flock($lckfh, LOCK_EX|LOCK_NB)) |
| 213 |
{ |
| 214 |
print "$0 is already running. Exiting.\n"; |
| 215 |
exit(1); |
| 216 |
} |
| 217 |
|
| 218 |
#if (-e $runningflag) |
| 219 |
#{ |
| 220 |
# die "Can't manage export; another process is already managing it.\n"; |
| 221 |
#} |
| 222 |
|
| 223 |
if (defined($binpath)) |
| 224 |
{ |
| 225 |
$binpath = "$binpath/$ENV{\"JSOC_MACHINE\"}"; |
| 226 |
} |
| 227 |
else |
| 228 |
{ |
| 229 |
$binpath = ""; |
| 230 |
} |
| 231 |
|
| 232 |
#local $ENV{"PATH"} = "$binpath:$ENV{\"PATH\"}"; |
| 233 |
#local $ENV{"PATH"} = "$scrpath:$ENV{\"PATH\"}"; |
| 234 |
local $ENV{"JSOCROOT"} = $root; |
| 235 |
local $ENV{"JSOC_DBUSER"} = $dbuser; |
| 236 |
local $ENV{"JSOC_DBNAME"} = $dbname; |
| 237 |
|
| 238 |
#`touch $runningflag`; |
| 239 |
`echo $$ > $runningflag`; |
| 240 |
$logfile = kExportDir . "/logs/" . `date +"%F_%R.log"`; |
| 241 |
open(LOG, ">> $logfile") || die "Couldn't open logfile '$logfile'.\n"; |
| 242 |
$daemonlog = kExportDir . "/logs/exportlog${logflag}.txt"; |
| 243 |
|
| 244 |
my($datenow) = `date`; |
| 245 |
chomp($datenow); |
| 246 |
$msg = "Started by $ENV{'USER'} at $datenow on machine $ENV{'HOST'} using $dbhost.\n"; |
| 247 |
print LOG $msg; |
| 248 |
|
| 249 |
my($rout); |
| 250 |
my($cmd); |
| 251 |
my($dlogfh); |
| 252 |
my($err) = 0; |
| 253 |
|
| 254 |
unless (GetDLogFH(\$dlogfh, $daemonlog)) |
| 255 |
{ |
| 256 |
print $dlogfh $msg; |
| 257 |
CloseDLog(\$dlogfh); |
| 258 |
} |
| 259 |
|
| 260 |
$cmd = "$binpath/$manage JSOC_DBHOST=$dbhost procser=$procser"; |
| 261 |
|
| 262 |
my($msgq) = {lastsend => time(), msgs => {}}; |
| 263 |
|
| 264 |
while (1) |
| 265 |
{ |
| 266 |
# print "running $cmd.\n"; |
| 267 |
$rout = qx($cmd 2>&1); |
| 268 |
|
| 269 |
if ($? == -1) |
| 270 |
{ |
| 271 |
QueueMessage($msgq, &kMsgType1, "Export Daemon Execution Failure!!", &kMailMessage1); |
| 272 |
} |
| 273 |
elsif ($? & 127) |
| 274 |
{ |
| 275 |
# jsoc_export_manage died in response to an unhandled signal |
| 276 |
my($sig) = $? & 127; |
| 277 |
|
| 278 |
QueueMessage($msgq, &kMsgType1, "Export Daemon Execution Failure!!", &kMailMessage2, "DB Host: $dbhost\n", "Unhandled signal: $sig.\n"); |
| 279 |
} |
| 280 |
elsif (($? >> 8) != 0) |
| 281 |
{ |
| 282 |
# jsoc_export_manage returned with an error code |
| 283 |
$msg = "$manage returned with a non-zero code of $? >> 8.\n"; |
| 284 |
unless (GetDLogFH(\$dlogfh, $daemonlog)) |
| 285 |
{ |
| 286 |
print $dlogfh $msg; |
| 287 |
} |
| 288 |
|
| 289 |
print LOG $msg; |
| 290 |
} |
| 291 |
|
| 292 |
if (defined($rout) && length($rout) > 0) |
| 293 |
{ |
| 294 |
$msg = "$rout\n"; |
| 295 |
unless (GetDLogFH(\$dlogfh, $daemonlog)) |
| 296 |
{ |
| 297 |
print $dlogfh $msg; |
| 298 |
} |
| 299 |
|
| 300 |
print LOG $msg; |
| 301 |
} |
| 302 |
|
| 303 |
SendPendingMessages($msgq); |
| 304 |
|
| 305 |
CloseDLog(\$dlogfh); |
| 306 |
|
| 307 |
if (KeepRunning($runningflag)) |
| 308 |
{ |
| 309 |
sleep(2); |
| 310 |
} |
| 311 |
else |
| 312 |
{ |
| 313 |
last; |
| 314 |
} |
| 315 |
} |
| 316 |
|
| 317 |
$msg = "Stopped by $ENV{'USER'} at " . `date` . ".\n"; |
| 318 |
unless (GetDLogFH(\$dlogfh, $daemonlog)) |
| 319 |
{ |
| 320 |
print $dlogfh $msg; |
| 321 |
CloseDLog(\$dlogfh); |
| 322 |
} |
| 323 |
|
| 324 |
print LOG $msg; |
| 325 |
close(LOG); |
| 326 |
|
| 327 |
# Don't leave junk laying about |
| 328 |
CleanRunFlag($runningflag); |
| 329 |
|
| 330 |
# release the exclusive file lock |
| 331 |
flock($lckfh, LOCK_UN); |
| 332 |
$lckfh->close; |
| 333 |
|
| 334 |
exit($err); |
| 335 |
|
| 336 |
# END |
| 337 |
sub IOwnRunFlag |
| 338 |
{ |
| 339 |
my($file) = $_[0]; |
| 340 |
my($fexists); |
| 341 |
my($iownit); |
| 342 |
my($line); |
| 343 |
|
| 344 |
$fexists = (-e $file); |
| 345 |
if ($fexists) |
| 346 |
{ |
| 347 |
if (open(FLFILE, "<$file")) |
| 348 |
{ |
| 349 |
$line = <FLFILE>; |
| 350 |
chomp($line); |
| 351 |
$iownit = ($line == $$); |
| 352 |
close(FLFILE); |
| 353 |
} |
| 354 |
} |
| 355 |
|
| 356 |
return $fexists && $iownit; |
| 357 |
} |
| 358 |
|
| 359 |
sub KeepRunning |
| 360 |
{ |
| 361 |
my($file) = $_[0]; |
| 362 |
|
| 363 |
return IOwnRunFlag($file) |
| 364 |
} |
| 365 |
|
| 366 |
sub CleanRunFlag |
| 367 |
{ |
| 368 |
my($file) = $_[0]; |
| 369 |
|
| 370 |
if (IOwnRunFlag($file)) |
| 371 |
{ |
| 372 |
unlink($file); |
| 373 |
} |
| 374 |
} |
| 375 |
|
| 376 |
sub GetDLogFH |
| 377 |
{ |
| 378 |
my($rfh) = shift; # reference to filehandle object |
| 379 |
my($dlog) = shift; |
| 380 |
my($msgq) = shift; |
| 381 |
my($err); |
| 382 |
|
| 383 |
$err = 0; |
| 384 |
|
| 385 |
if (!defined($$rfh)) |
| 386 |
{ |
| 387 |
$$rfh = FileHandle->new(">>$dlog"); |
| 388 |
|
| 389 |
if (!defined($$rfh)) |
| 390 |
{ |
| 391 |
QueueMessage($msgq, &kMsgType1, "Export Daemon Log Unavailable", &kMailMessage3); |
| 392 |
$err = 1; |
| 393 |
} |
| 394 |
} |
| 395 |
|
| 396 |
return $err; |
| 397 |
} |
| 398 |
|
| 399 |
sub CloseDLog |
| 400 |
{ |
| 401 |
my($rfh) = $_[0]; # reference to filehandle object |
| 402 |
|
| 403 |
if (defined($$rfh)) |
| 404 |
{ |
| 405 |
$$rfh->close(); |
| 406 |
undef($$rfh); |
| 407 |
} |
| 408 |
} |
| 409 |
|
| 410 |
sub SendPendingMessages |
| 411 |
{ |
| 412 |
my($msgs) = shift; |
| 413 |
my($imsg); |
| 414 |
my($msg); |
| 415 |
my($subj); |
| 416 |
|
| 417 |
if (time() - $msgs->{lastsend} > &kMsgQSendInterval) |
| 418 |
{ |
| 419 |
# Check for pending messages |
| 420 |
foreach $imsg (keys(%{$msgq->{msgs}})) |
| 421 |
{ |
| 422 |
$msg = $msgq->{msgs}->{$imsg}->{msg}; |
| 423 |
$subj = $msgq->{msgs}->{$imsg}->{subj}; |
| 424 |
open(MAILPIPE, "| /bin/mail -s \"$subj\" " . &kMailList) || die "Couldn't open 'mail' pipe.\n"; |
| 425 |
print MAILPIPE $msg; |
| 426 |
close(MAILPIPE); |
| 427 |
|
| 428 |
if ($msgq->{msgs}->{$imsg}->{ntimes} > 1) |
| 429 |
{ |
| 430 |
$msgq->{msgs}->{$imsg}->{ntimes} = $msgq->{msgs}->{$imsg}->{ntimes} - 1; |
| 431 |
} |
| 432 |
else |
| 433 |
{ |
| 434 |
delete($msgq->{msgs}->{$imsg}); |
| 435 |
} |
| 436 |
} |
| 437 |
|
| 438 |
$msgs->{lastsend} = time(); |
| 439 |
} |
| 440 |
} |
| 441 |
|
| 442 |
# Message queue: |
| 443 |
# key - id |
| 444 |
# val - hash : {instime => 10292392, msg => "export failure", ntimes => 5} |
| 445 |
# where instime is unix seconds identifying time message was inserted into queue. |
| 446 |
# subj is the mail subject |
| 447 |
# msg is the message to mail |
| 448 |
# ntimes is the number of times to send message out. |
| 449 |
sub QueueMessage |
| 450 |
{ |
| 451 |
my($msgq) = shift; |
| 452 |
my($type) = shift; |
| 453 |
my($subj) = shift; |
| 454 |
my(@msg) = @_; |
| 455 |
my($oktoins); |
| 456 |
|
| 457 |
if (exists($msgq->{msgs}->{$type})) |
| 458 |
{ |
| 459 |
$oktoins = time() - $msgq->{msgs}->{$type}->{instime} > &kMsgQInterval; |
| 460 |
|
| 461 |
# Message already exists. Don't add to queue until some time elapses. |
| 462 |
if ($oktoins) |
| 463 |
{ |
| 464 |
$msgq->{msgs}->{$type}->{ntimes} = $msgq->{msgs}->{$type}->{ntimes} + 1; |
| 465 |
} |
| 466 |
} |
| 467 |
else |
| 468 |
{ |
| 469 |
my($msgstr) = join('', @msg); |
| 470 |
$msgq->{msgs}->{$type} = {instime => time(), subj => $subj, msg => $msgstr, ntimes => 1}; |
| 471 |
} |
| 472 |
} |
| 473 |
|
| 474 |
__DATA__ |
