ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/JSOC/proj/util/scripts/jsoc_update.pl
Revision: 1.13
Committed: Fri Feb 7 22:52:25 2014 UTC (9 years, 7 months ago) by arta
Content type: text/plain
Branch: MAIN
CVS Tags: Ver_9-1, Ver_LATEST, Ver_9-3, Ver_9-41, Ver_9-2, Ver_8-8, Ver_8-6, Ver_8-7, Ver_8-4, Ver_8-5, Ver_9-5, Ver_9-4, Ver_8-10, Ver_8-11, Ver_8-12, Ver_9-0, HEAD
Changes since 1.12: +19 -3 lines
Log Message:
Make sure that jsoc_update.pl has access to cvsupdate.log. Also, clean up output that gets written when the call to jsoc_sync.pl fails.

File Contents

# Content
1 #!/home/jsoc/bin/linux_x86_64/activeperl -w
2
3
4 use File::stat;
5 use Fcntl qw(:mode);
6 use Getopt::Long;
7 use FileHandle;
8 use Sys::Hostname;
9 use Cwd qw(realpath chdir); # OMG! Need to override chdir, otherwise $ENV{PWD} is NOT
10 # updated when chdir is called.
11 use FindBin qw($Bin);
12 use lib ($Bin, "$Bin/../../../base/libs/perl");
13 use drmsArgs;
14
15 use constant kSuccess => 0;
16 use constant kBadArgs => 1;
17 use constant kCantSSH => 2;
18 use constant kConflicts => 3;
19 use constant kFileStatus => 4;
20 use constant kCantExeOnMach => 5;
21 use constant kCantUpdateSrc => 6;
22
23
24 use constant kUpdateLog => "cvsupdate.log";
25 use constant kMtabFile => "/etc/mtab";
26 use constant kTmpDir => "/tmp";
27
28 my(@machines) =
29 (
30 "n02",
31 "solar3"
32 );
33
34 my($optsinH);
35 my($opts);
36 my($optdir);
37 my($machlist);
38 my($cvstree);
39 my($jsocroot);
40 my($nfsf);
41 my($volume);
42 my($mount);
43 my($netpath);
44 my($mach);
45 my($hostnm);
46
47
48 # If the caller is attempting to update a CVS tree that resides on a local disk, then
49 # it MIGHT not be possible to update that tree after ssh'ing to a sanctioned machine
50 # (e.g., n02). Or the caller might be attempting to update a CVS tree that resides
51 # on a network disk, but the disk might not be exported to a sanctioned machine.
52 # This script checks for those problems.
53 #
54 # Determine whether the cvs tree resides on a local disk, or if it is NFS-
55 # mounted. If it is on local disk, then create a network volume name for the disk (e.g.,
56 # <machine>:<path>). If the tree is NFS-mounted, then use the mtab to create a
57 # network volume.
58 #
59 # If the CVS tree does not reside on a disk accessible from the sanctioned machines,
60 # ask the caller if they want to build the updated tree on the machine running this
61 # script.
62
63 $optsinH =
64 {
65 "dir" => 's',
66 "machs" => 's'
67 };
68
69 $opts = new drmsArgs($optsinH, 0);
70
71 if (!defined($opts))
72 {
73 exit(kBadArgs);
74 }
75
76 $optdir = $opts->Get("dir");
77 $machlist = $opts->Get("machs");
78
79 # If user did not specify a directory to update, use the one specified by the JSOCROOT
80 # env variable, if it exists.
81 if (!defined($optdir))
82 {
83 $jsocroot = $ENV{'JSOCROOT'};
84 (!defined($jsocroot)) ? $cvstree = $ENV{'PWD'} : $cvstree = $jsocroot;
85 }
86 else
87 {
88 # $cvstree needs to be an absolute path. If it isn't, make it so.
89 if ($optdir =~ /^\s*\//)
90 {
91 $cvstree = $optdir;
92 }
93 else
94 {
95 $cvstree = "$ENV{'PWD'}/$optdir"
96 }
97 }
98
99 if (defined($machlist))
100 {
101 my(@mintermed) = split(/,/, $machlist);
102 @machines = map({ ($_ =~ /\s*(\S+)\s*/)[0] } @mintermed);
103
104 }
105
106 # Determine if the cvs tree specified exists
107 if (!(-d $cvstree))
108 {
109 print STDERR "Specified CVS tree does not exist.\n";
110 exit(kBadArgs);
111 }
112
113 # Determine if the cvs tree resides on a local disk.
114 # 1. Obtain the device number for each record in /etc/mtab.
115 # 2. Walk down this list, comparing the device on which the CVS tree resides with each
116 # record in this list. If there is a match, then the CVS tree resides on a mounted directory.
117 # Check the matching mtab record for an NFS file system. If not, then assume
118 # the CVS tree is on a local disk. Otherwise, it is NFS-mounted, and we should
119 # obtain the hosting volume and mount point from the mtab record.
120 $nfsf = (GetNFSInfo($cvstree, \$volume, \$mount));
121
122 if (!$nfsf)
123 {
124 # Assume the tree is on a local disk. Create a 'network name' for the tree.
125 $hostnm = hostname();
126 $netpath = "$hostnm:" . realpath($cvstree);
127 }
128 else
129 {
130 $netpath = realpath($cvstree);
131 $netpath =~ s/$mount/$volume/;
132 }
133
134 # Take the local path of the JSOC tree, identify the mount root in this path
135 # (the mount root is the part of the path that resides in the appropriate mtab
136 # record), then substitute the mount-path part of the local path with the
137 # network volume (e.g., <machine>:<path>). So if the JSOC tree's local path
138 # is /auto/home1/arta/jsoctrees/JSOC, and the mtab record for the network
139 # drive that contains this tree is "sunroom:/home1 /auto/home1 ..."
140 # substitute "/auto/home1" in the JSOC-tree path with "sunroom:/home1" to
141 # form the network path "sunroom:/home1/arta/jsoctrees/JSOC".
142
143 print "CVS JSOC tree being updated ==> $netpath\n";
144
145 # Build the JSOC tree on the sanctioned machines, if it is mounted.
146
147 # Anonymous hash whose keys are machines. For each key, the value is the
148 # path, local to the machine, to the CVS tree being updated.
149 my($mpaths) = GetLocalPaths($netpath, @machines);
150 my(@mpkeys) = keys(%$mpaths);
151 my($impkey);
152 my($line);
153 my($cmd);
154
155 if ($#mpkeys == -1)
156 {
157 $hostnm = hostname();
158 print "The CVS tree $netpath is not mounted on any of the requested build machines (i.e., n02).\n";
159 print "Would you like to build on the machine on which this script is running, $hostnm? (y/n)\n";
160 $line = <STDIN>;
161 chomp($line);
162 $line = lc($line);
163
164 if ($line =~ /^y/)
165 {
166 $mpaths->{$hostnm} = $cvstree;
167 push(@mpkeys, $hostnm);
168 print " Local path ==> $hostnm:$cvstree.\n";
169 }
170 }
171
172 if ($#mpkeys >= 0)
173 {
174 my($cdir) = realpath($ENV{'PWD'});
175 my(@rsp);
176 my($rspstr);
177
178 print "\n";
179 print "#############################################\n";
180 print "####### Starting CVS update #################\n";
181 print "#############################################\n";
182 print "##\n";
183 print "## changing working directory to $cvstree.\n";
184
185 chdir($cvstree);
186 unless (UpdateTree($cvstree))
187 {
188 # Run the configure script - should really check the return code.
189 print "####### Running configure script ############\n";
190 @rsp = `./configure 2>&1`;
191 print "## Done (output from configure follows).\n";
192 print "## ";
193 $rspstr = join("## ", @rsp);
194 print $rspstr;
195
196 # Do the build on each machine
197 print "####### Building binaries on machines #####\n";
198 map({ BuildOnMachine($_, $mpaths); } @mpkeys);
199 print "## Done building.\n";
200 print "##\n";
201 }
202 else
203 {
204 # Failure updating CVS tree.
205 print "## restoring working directory to $cdir.\n";
206 chdir($cdir);
207
208 print "###########################################\n";
209 print "####### Update of CVS binaries FAILED!! ###\n";
210 print "###########################################\n";
211
212 exit(kCantUpdateSrc);
213 }
214
215 print "## restoring working directory to $cdir.\n";
216 chdir($cdir);
217
218 print "###########################################\n";
219 print "####### Update of CVS binaries complete. ##\n";
220 print "###########################################\n";
221 }
222
223 exit(kSuccess);
224
225 #my($st) = stat("/tmp"); # 2050
226 #my($st) = stat("/SUM12"); # 33 (must refer to the physical disk device on d02)
227 #my($st) = stat("/auto/SUM12"); # 33
228 #my($st) = stat("/dev/sda2"); # 2050
229 #my($st) = stat("/auto/home1");
230 #print "isblock\n" if (S_ISBLK($st->mode));
231 #(S_ISBLK($st->mode)) ? print $st->rdev . "\n" : print $st->dev . "\n";
232
233 sub Usage
234 {
235 print "jsoc_update.pl\n";
236 print " If the environment variable JSOCROOT is set, update the CVS tree specified by\n";
237 print " \$JSOCROOT. Otherwise, if the current directory is a valid DRMS CVS tree, update\n";
238 print " the current directory. If the current directory is not a valid DRMS CVS tree, exit\n";
239 print " without updating anything.\n";
240
241 print "jsoc_update -d <path>";
242 print " Update the DRMS CVS tree specified by <path>.\n";
243 }
244
245 # Returns 1 and the network volume containing the JSOC tree (if the JSOC tree is
246 # NFS-mounted). Returns 0 otherwise.
247 sub GetNFSInfo
248 {
249 my($tree) = $_[0];
250 my($volr) = $_[1];
251 my($mountr) = $_[2];
252
253 my($rv) = 0;
254 my(@mtab);
255 my($fh);
256
257 # Get the device number for the CVS tree.
258 my($st) = stat($tree);
259 my($devno) = (S_ISBLK($st->mode)) ? $st->rdev : $st->dev; # number of device containing
260 # JSOC tree.
261 my($info);
262
263 # Open mtab file
264 $fh = FileHandle->new("<" . kMtabFile);
265
266 if (defined($fh))
267 {
268 @mtab = <$fh>;
269
270 # Not sure how to short-circuit the map function (we don't want to examine every line
271 # in the mtab if we found a relevant one).
272 map({ ProcessMtabInfo(\$info, $devno, $_); } @mtab);
273 $fh->close();
274 }
275 else
276 {
277 print STDERR "Fatal error - cannot find mtab.\n";
278 }
279
280 if (defined($info))
281 {
282 $rv = 1;
283 $$volr = $info->{'vol'};
284 $$mountr = $info->{'mount'};
285 }
286
287 return $rv;
288 }
289
290 # Find the network drive that contains the JSOC tree.
291 sub ProcessMtabInfo
292 {
293 my($infor) = $_[0];
294 my($devno) = $_[1]; # number of the device containing the JSOC tree
295 my($line) = $_[2];
296
297 my($vol);
298 my($mount);
299 my($type);
300 my($st);
301 my($mtabdevno);
302
303 chomp($line);
304
305 # 1 - NFS volume (machine:volume)
306 # 2 - mount point
307 # 3 - partition type
308 if ($line =~ /\s*(\S+)\s*(\S+)\s*(\S+)/)
309 {
310 $vol = $1;
311 $mount = $2;
312 $type = $3;
313
314 # Look up the device number for this line
315 $st = stat($mount);
316 if (defined($st))
317 {
318 $mtabdevno = (S_ISBLK($st->mode)) ? $st->rdev : $st->dev;
319
320 if ($devno == $mtabdevno && $type =~ /nfs/i)
321 {
322 # match - we have the device on which the CVS tree resides, and that device
323 # is NFS-mounted.
324 if (!(defined($$infor)))
325 {
326 # New empty hash.
327 $$infor = {};
328 }
329
330 # Save the network drive that contains the JSOC tree (and where it is mounted on
331 # the current machine).
332 $$infor->{'vol'} = $vol;
333 $$infor->{'mount'} = $mount;
334 }
335 }
336 }
337 }
338
339 # $line is the remote machine's (e.g., n02) mtab line.
340 sub ExtractMachPath
341 {
342 my($netpath) = $_[0];
343 my($line) = $_[1];
344
345 my($volume);
346 my($mount);
347
348 # First, extract the volume from the mtab record
349 chomp($line);
350
351 if ($line =~ /^\s*(\S+)\s+(\S+)/)
352 {
353 $volume = $1;
354 $mount = $2;
355
356 # $volume - like sunroom:/home0
357 # $1 - like /home0
358 # $netpath - like sunroomg:/home0/arta/jsoctrees/JSOC
359 if ($netpath =~ /^$volume(.+)/)
360 {
361 return $mount . $1;
362 }
363 else
364 {
365 # There might be a trailing 'g' on the $netpath (legacy - used to mean
366 # gigabit for a gigabit network). But all networks are at least Gb now.
367 # One physical drive might have a network name whose symbolic name ends
368 # in a 'g' on one machine, but not on another. Try to remove the trailing
369 # 'g' before doing a comparison. The mtab line might also have this trailing
370 # 'g'.
371 #
372 # $netpath2 - like sunroom:/home0/arta/jsoctrees/JSOC
373 my($netpath2) = $netpath;
374
375 $netpath2 =~ s/g:/:/;
376 $volume =~ s/g:/:/;
377
378 if ($netpath2 =~ /^$volume(.+)/)
379 {
380 return $mount . $1;
381 }
382 else
383 {
384 return ();
385 }
386 }
387 }
388 else
389 {
390 print STDERR "Unexpected response from server: $line\n";
391 return ();
392 }
393 }
394
395 sub GetLocalPaths
396 {
397 my($netpath) = shift; # network path to CVS tree
398 my(@machines) = @_;
399
400 my($rv);
401 my(@machpath);
402
403 foreach $mach (@machines)
404 {
405 # make a test connection
406 if (open(STATCMD, "(ssh $mach cat /etc/mtab) 2>&1 |"))
407 {
408 my(@remotemtab) = <STATCMD>;
409
410 close(STATCMD);
411
412 # Extract record for $volume.
413 @machpath = map({ ExtractMachPath($netpath, $_); } @remotemtab);
414
415 my($test) = $#machpath;
416
417 if ($#machpath == 0)
418 {
419 # CVS tree lives on $mach at $machpath[0];
420 print " Local path ==> $mach:$machpath[0].\n";
421
422 if (!defined($rv))
423 {
424 $rv = {};
425 }
426
427 $rv->{$mach} = $machpath[0];
428 }
429 else
430 {
431 print "CVS tree $netpath is not mounted on $mach; skipping machine.\n";
432 next;
433 }
434 }
435 else
436 {
437 print "Cannot ssh to $mach.\n";
438 exit(kCantSSH);
439 }
440 }
441
442 return $rv;
443 }
444
445 sub UpdateTree
446 {
447 my($cvstree) = $_[0];
448
449 my($rv) = 0;
450 my(@statchk);
451 my($rsp);
452 my(@splitRes);
453
454 print "####### Downloading repository changes ######\n";
455
456 # Update local CVS tree with CVS-repository changes.
457 $cmd = "$Bin/jsoc_sync.pl -l" . kUpdateLog; # use the jsoc_sync.pl relative to this script
458 print "## Updating source files in $cvstree (calling '$cmd').\n";
459 $rsp = qx($cmd 2>&1);
460
461 if ($? >> 8)
462 {
463 # Error calling jsoc_sync.pl.
464 print STDERR "## Failure calling jsoc_sync.pl\n";
465 }
466
467 @splitRes = split(qr/\n/, $rsp);
468 $rsp = "";
469 foreach my $line (@splitRes)
470 {
471 $rsp = $rsp . "## $line\n";
472 }
473
474 print "$rsp";
475 print "##\n";
476
477 # Log is in parent directory.
478 if (open(UDL, "<" . "../" . kUpdateLog))
479 {
480 my(@content) = <UDL>;
481 my(@conflicts) = grep({ ($_ =~ /^\s*C/) ? $_ : () } @content);
482 my($iline);
483
484 close(UDL);
485
486 if ($#conflicts >= 0)
487 {
488 print "## A list of locally modified files with changes that conflict with repository changes follows:\n";
489 foreach $iline (@conflicts)
490 {
491 print "## $iline\n";
492 }
493
494 print "## Please resolve these conflicts, then update again.\n";
495 print "## (see release notes to learn how to deal with conflicting file changes.)\n";
496 exit(kConflicts);
497 }
498
499 print "####### Checking file status ################\n";
500 @content = CheckStatus();
501 print "## Done.\n";
502
503 if ($#content >= 0)
504 {
505 print "## A list of local files that differ from their repository counterparts follows:\n";
506 foreach $iline (@content)
507 {
508 chomp($iline);
509 print "## $iline\n";
510 }
511
512 print "##\n";
513 }
514
515 print "## All checks succeeded. Continue by entering 'cont'.\n";
516 print "## ";
517 $line = <STDIN>;
518 chomp($line);
519 $line = lc($line);
520
521 if ($line !~ /^\s*cont/)
522 {
523 print "## Update aborted by user.\n";
524 $rv = 1;
525 }
526 }
527 else
528 {
529 # Couldn't open update log - bail.
530 print "## Could not open update log file ../" . kUpdateLog . ".\n";
531 $rv = 1;
532 }
533
534 print "##\n";
535
536 return $rv;
537 }
538
539 sub CheckStatus
540 {
541 my(@res);
542
543 my $PID = getppid;
544 my $user = $ENV{'USER'};
545 my $logfile = "/tmp/cvs_status_".$user."_$PID.log";
546 `cvs status 1> $logfile 2>&1`;
547 open(CV, $logfile) || die "Can't open $logfile: $!\n";
548
549 while (<CV>)
550 {
551 #print "$_"; #!!!TEMP
552 if (/^File:/)
553 {
554 if (/Up-to-date/)
555 {
556 next;
557 }
558 push(@res, "$_");
559 <CV>;
560 <CV>;
561 $_ = <CV>; #get Repository revision line
562 s/\s+/ /g; #compress multiple spaces
563 push(@res, "$_\n");
564 }
565 }
566
567 return @res;
568 }
569
570 sub BuildOnMachine
571 {
572 my($mach) = $_[0];
573 my($mpathsr) = $_[1];
574
575 my($plat);
576 my($lpath) = $mpathsr->{$mach};
577
578 if (defined($lpath))
579 {
580 # Should really capture the return code from this ssh cmd.
581 if (open(CMD, "(ssh $mach " . "\'" . "echo \$JSOC_MACHINE" . "\'" . ") 2>&1 |"))
582 {
583 $plat = <CMD>;
584 chomp($plat);
585 close(CMD);
586
587 print "## Starting build on platform $plat.\n";
588
589 # Should really capture return code from this ssh cmd.
590 if (open(CMD, "(ssh $mach 'cd $lpath; /home/jsoc/make_jsoc.pl') 1>make_jsoc_$plat.log 2>&1 |"))
591 {
592 close(CMD);
593 }
594 else
595 {
596 print "## Unable to run cmd on $mach [1].\n";
597 }
598
599 print "## Build on platform $plat complete.\n";
600 }
601 else
602 {
603 print "## Unable to run cmd on $mach [2].\n";
604 }
605 }
606 }