ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/JSOC/proj/util/scripts/findsessionrecs.pl
Revision: 1.12
Committed: Wed Nov 12 05:20:21 2014 UTC (8 years, 10 months ago) by arta
Content type: text/plain
Branch: MAIN
CVS Tags: Ver_8-8, Ver_8-11, Ver_8-10, Ver_LATEST, Ver_9-41, Ver_8-12, Ver_8-7, Ver_9-5, Ver_9-4, Ver_9-3, Ver_9-2, Ver_9-1, Ver_9-0, HEAD
Changes since 1.11: +7 -1 lines
Log Message:
Skip hmi.cosmic_rays - too slow.

File Contents

# Content
1 #!/home/jsoc/bin/linux_x86_64/activeperl
2
3 # NOTE - THIS WAS MY FIRST ATTEMPT AT GETTING THIS TO WORK IN THE FACE OF HAVING REMOVED EARLY drms_session LOGS. THE
4 # QUERIES IN CacheRecs() RUN TOO SLOWLY THOUGH. THERE IS NO WAY TO JOIN THE SERIES TABLES WITH THE drms_session TABLES
5 # IN A WAY THAT RUNS QUICK ENOUGH. FOR ONE TABLE WITH 100M RECORDS, I LET THE SECOND QUERY RUN ABOUT 14 HOURS BEFORE
6 # TERMINATING IT.
7
8 # For each series in the specified namespaces, the script iterates through all records in the series and determines
9 # whether each one was created within the crash window. For each record, the sessionid is used to find the relevant
10 # record in the drms_session table. Then the starttime and endtime values are compared to the bcrash and ecrash
11 # values. If the record was created during the crash window, then it gets reported in a list of records that were
12 # created during the crash window.
13
14 use warnings;
15 use strict;
16
17 use POSIX qw(ceil floor);
18 use DBI;
19 use DBD::Pg;
20 use Data::Dumper;
21 use DateTime::Format::Strptime;
22 use FindBin qw($Bin);
23 use lib ("$Bin/../../../base/libs/perl");
24 use drmsArgs;
25
26 use constant DEBUG_ON => 0;
27
28 use constant kRetSuccess => 0;
29 use constant kRetInvalidArgs => 1;
30 use constant kRetDbQuery => 2;
31 use constant kRetFileIO => 3;
32
33 use constant kArgDbname => "dbname";
34 use constant kArgDbhost => "dbhost";
35 use constant kArgDbport => "dbport";
36 use constant kArgExclude => "exclude"; # A list of namespaces that contain series to exclude from examination.
37 use constant kArgInclude => "include";
38 use constant kArgSeries => "series"; # A list of series to examine. Overrides the exclude and include parameters.
39 use constant kArgBcrash => "bcrash";
40 use constant kArgEcrash => "ecrash";
41 use constant kArgFileOut => "out";
42
43 # Globals
44 my($gRecCache) = {};
45
46 my($argsinH);
47 my($optsinH);
48 my($args);
49 my($opts);
50 my($dbname);
51 my($dbhost);
52 my($dbport);
53 my($excludeList);
54 my(@excludedNs);
55 my($exclude);
56 my($ignoreExcInc);
57 my($bcrash);
58 my($includeList);
59 my(@includedNs);
60 my($seriesList);
61 my(@series);
62 my($ecrash);
63 my($fileout);
64 my($fh);
65 my($dbuser);
66 my($dsn);
67 my($dbh);
68 my($nspace);
69 my($table);
70 my($stmnt);
71 my($rows);
72 my(@rowarr);
73 my($rowstab);
74 my(@rowarrtab);
75 my($firstrec);
76 my($lastrec);
77 my($err);
78 my($rv);
79
80 $rv = &kRetSuccess;
81
82 $argsinH =
83 {
84 &kArgDbname => 's',
85 &kArgDbhost => 's',
86 &kArgDbport => 'i',
87 &kArgExclude => 's', # make this required so that people don't accidentally request processing of
88 # 43K series!
89 &kArgBcrash => 's',
90 &kArgEcrash => 's',
91 &kArgFileOut => 's'
92 };
93
94 $optsinH =
95 {
96 &kArgInclude => 's', # Inlcude applies after Exclude has filtered-out series.
97 &kArgSeries => 's' # If provided, then process these series only, and ignore the exclude/include flags.
98 };
99
100 $args = new drmsArgs($argsinH, 1);
101 $opts = new drmsArgs($optsinH, 0);
102
103 if (!defined($args) || !defined($opts))
104 {
105 $rv = &kRetInvalidArgs;
106 }
107 else
108 {
109 $dbname = $args->Get(&kArgDbname);
110 $dbhost = $args->Get(&kArgDbhost);
111 $dbport = $args->Get(&kArgDbport);
112 $excludeList = $args->Get(&kArgExclude);
113 $includeList = $opts->Get(&kArgInclude);
114 $seriesList = $opts->Get(&kArgSeries);
115 $bcrash = $args->Get(&kArgBcrash);
116 $ecrash = $args->Get(&kArgEcrash);
117 $fileout = $args->Get(&kArgFileOut);
118 $dbuser = $ENV{USER};
119
120 @excludedNs = split(/,|\s+/, $excludeList);
121 if (defined($includeList))
122 {
123 @includedNs = split(/,|\s+/, $includeList);
124 }
125
126 if (defined($seriesList))
127 {
128 @series = split(/,|\s+/, $seriesList);
129 }
130
131 if (open($fh, ">$fileout"))
132 {
133 $dsn = "dbi:Pg:dbname=$dbname;host=$dbhost;port=$dbport";
134 $dbh = DBI->connect($dsn, $dbuser, ''); # will need to put pass in .pg_pass
135 }
136 else
137 {
138 print STDERR "Unable to open output file '$fileout' for writing.\n";
139 $rv = &kRetFileIO;
140 }
141
142 if (!$rv && defined($dbh))
143 {
144 # Get a list of all series we want to investigate
145 $err = 0;
146 $ignoreExcInc = 0;
147
148 if ($#series < 0)
149 {
150 # No series specified on command line. Search through namespaces.
151 $stmnt = "SELECT seriesname FROM drms_series()";
152 $rows = $dbh->selectall_arrayref($stmnt, undef);
153 $err = (NoErr($rows, \$dbh, $stmnt) == 1) ? 0 : 1;
154
155 if (!$err)
156 {
157 @rowarr = @$rows;
158 @series = map({$_->[0]} @rowarr);
159 }
160 }
161 else
162 {
163 $ignoreExcInc = 1;
164 }
165
166 if (!$err)
167 {
168 my($seriesRecs);
169
170 foreach my $aseries (@series)
171 {
172 ($nspace, $table) = ($aseries =~ /(\S+)\.(\S+)/);
173
174 $exclude = 0;
175
176 if ($ignoreExcInc)
177 {
178 # Ignore @excludledNs and @includedNs. Process all series in @series.
179 }
180 else
181 {
182 if ($#excludedNs >= 0)
183 {
184 foreach my $iexc (@excludedNs)
185 {
186 if ($nspace =~ /$iexc/)
187 {
188 $exclude = 1;
189 last;
190 }
191 }
192 }
193
194 if (!$exclude)
195 {
196 if ($#includedNs >= 0)
197 {
198 $exclude = 1;
199 foreach my $iinc (@includedNs)
200 {
201 if ($nspace =~ /$iinc/)
202 {
203 $exclude = 0;
204 last;
205 }
206 }
207 }
208 }
209 }
210
211 if ($exclude)
212 {
213 print STDERR "skipping series $aseries - it was excluded.\n";
214 next;
215 }
216
217 print STDERR "Processing series $aseries...\n";
218
219 if (lc($aseries) eq 'hmi.cosmic_rays')
220 {
221 print STDERR "skipping series $aseries - it is too big to analyze.\n";
222 next;
223 }
224
225 $seriesRecs = FindRecs($dbh, $aseries, $bcrash, $ecrash);
226
227 # Clear record cache so we do not use up memory.
228 PurgeRecs($aseries);
229
230 if (!defined($seriesRecs))
231 {
232 # No records in series - onto next series.
233 print STDERR "skipping series $aseries - there were no records created during the crash window.\n";
234 next;
235 }
236
237 if (scalar(@$seriesRecs) > 0)
238 {
239 # At least one output line - print a header.
240 print "seriesname\trecnum\tsunum\n";
241 print $fh "seriesname\trecnum\tsunum\n";
242 }
243
244 foreach my $rec (@$seriesRecs)
245 {
246 print "$rec->[0]\t$rec->[1]\t$rec->[2]\n";
247 print $fh "$rec->[0]\t$rec->[1]\t$rec->[2]\n";
248 }
249 } # series loop
250 }
251 else
252 {
253 $rv = kRetDbQuery;
254 }
255 }
256
257 if (defined($fh))
258 {
259 $fh->close();
260 }
261
262 exit $rv;
263 }
264
265 sub NoErr
266 {
267 my($rv) = $_[0];
268 my($dbh) = $_[1];
269 my($stmnt) = $_[2];
270 my($ok) = 1;
271
272 if (!defined($rv) || !$rv)
273 {
274 if (defined($$dbh) && defined($$dbh->err))
275 {
276 print STDERR "Error " . $$dbh->errstr . ": Statement '$stmnt' failed.\n";
277 }
278
279 $ok = 0;
280 }
281
282 return $ok;
283 }
284
285 sub GetSessionInfo
286 {
287 my($dbh, $table, $recno) = @_;
288 my($stmnt);
289 my($rows);
290 my(@rowarr);
291 my($err);
292 my(@rv);
293
294 # Get ns and session id from series table.
295 $stmnt = "SELECT sessionns, sessionid from $table WHERE recnum = $recno";
296 $rows = $dbh->selectall_arrayref($stmnt, undef);
297 $err = (NoErr($rows, \$dbh, $stmnt) == 1) ? 0 : 1;
298
299 if (!$err)
300 {
301 @rowarr = @$rows;
302
303 if ($#rowarr != 0)
304 {
305 $err = 1;
306 }
307 else
308 {
309 @rv = ($rowarr[0]->[0], $rowarr[0]->[1]);
310 }
311 }
312
313 return @rv;
314 }
315
316 sub HasSession
317 {
318 my($dbh, $table, $recno) = @_;
319
320 my($stmnt);
321 my($rows);
322 my(@rowarr);
323 my($ns);
324 my($sessionid);
325 my($err);
326 my($rv);
327
328 $err = 0;
329
330 # We do NOT want to cache any of these records. We are testing DRMS records identified by b-searching so we are hopping around
331 # lots of records. Anything we cached would be repreatedly flushed. Plus caching takes a long time.
332 ($ns, $sessionid) = GetSessionInfo($dbh, $table, $recno);
333
334 if (defined($ns) && defined($sessionid))
335 {
336 # See if a session exists for this series' record.
337 $stmnt = "SELECT sessionid FROM $ns\.drms_session WHERE sessionid = $sessionid";
338
339 $rows = $dbh->selectall_arrayref($stmnt, undef);
340 $err = (NoErr($rows, \$dbh, $stmnt) == 1) ? 0 : 1;
341
342 if (!$err)
343 {
344 @rowarr = @$rows;
345
346 if ($#rowarr == 0)
347 {
348 $rv = 1;
349 }
350 else
351 {
352 $rv = 0;
353 }
354 }
355 }
356
357 return $rv;
358 }
359
360 sub GetTimes
361 {
362 my($dbh, $table, $recno, $nsIn, $sessionidIn) = @_;
363
364 my($stmnt);
365 my($rows);
366 my(@rowarr);
367 my($ns);
368 my($sessionid);
369 my($err);
370 my(@rv);
371
372 $err = 0;
373
374 # We do NOT want to cache any of these records. We are testing DRMS records identified by b-searching so we are hopping around
375 # lots of records. Anything we cached would be repreatedly flushed. Plus caching takes a long time.
376 if (defined($nsIn) && defined($sessionidIn))
377 {
378 $ns = $nsIn;
379 $sessionid = $sessionidIn;
380 }
381 else
382 {
383 ($ns, $sessionid) = GetSessionInfo($dbh, $table, $recno);
384 }
385
386 if (defined($ns) && defined($sessionid))
387 {
388 # See if a session exists for this series' record.
389 $stmnt = "SELECT starttime, endtime FROM $ns\.drms_session WHERE sessionid = $sessionid";
390
391 $rows = $dbh->selectall_arrayref($stmnt, undef);
392 $err = (NoErr($rows, \$dbh, $stmnt) == 1) ? 0 : 1;
393
394 if (!$err)
395 {
396 @rowarr = @$rows;
397
398 if ($#rowarr == 0)
399 {
400 @rv = ($rowarr[0]->[0], $rowarr[0]->[1]);
401 }
402 }
403 }
404
405 return @rv;
406 }
407
408 # Given a $recno, find the next valid (recno exists in series table) in the direction specified by $down. The search will not find
409 # any record below/above $bound.
410 sub GetValidRec
411 {
412 my($dbh,
413 $series,
414 $recno,
415 $down,
416 $bound) = @_;
417
418 my($rv);
419 my($rows);
420 my(@rowarr);
421 my($err);
422
423 $err = 0;
424
425 if ($down)
426 {
427 $stmnt = "SELECT recnum FROM " . $series . " WHERE recnum >= " . $bound . " AND recnum <= " . $recno . " ORDER BY recnum DESC LIMIT 1";
428 }
429 else
430 {
431 $stmnt = "SELECT recnum FROM " . $series . " WHERE recnum >= " . $recno . " AND recnum <= " . $bound . " ORDER BY recnum ASC LIMIT 1";
432 }
433
434 $rows = $dbh->selectall_arrayref($stmnt, undef);
435 $err = (NoErr($rows, \$dbh, $stmnt) == 0);
436
437 if (!$err)
438 {
439 @rowarr = @$rows;
440
441 if ($#rowarr != 0)
442 {
443 $err = 1;
444 }
445 else
446 {
447 $rv = $rowarr[0]->[0];
448 }
449 }
450
451 return $rv;
452 }
453
454 # Returns an array of records (series, recnum, sunum). The first element in this array if the first record in the
455 # crash window, and the last element is the last record in this crash window. The array elements are sorted by
456 # recnum.
457
458 # Now that we are deleting old drms_session logs, we need to skip DRMS records that have no drms_session rows. It is not
459 # possible to evaluate all DRMS records, in a timely manner, to determine if they have drms_session rows or not. But
460 # we can b-search and find the first DRMS record that has a drms_session row. Use the first such DRMS record as the
461 # value for min.
462
463 sub FindRecs
464 {
465 my($dbh,
466 $series,
467 $bcrash,
468 $ecrash) = @_;
469 my($stable);
470 my($stmnt);
471 my($state) = "before"; # in - either first or last series record in crash window.
472 #
473 my($min); # first record in series.
474 my($max); # last record in series.
475 my($frec); # Lower bound for b-searching.
476 my($lrec); # Upper bound for b-searching.
477 my($loc);
478 my($locF); # location of record with minimum recnum.
479 my($locL); # location of record with maximum recnum.
480 my($rec); # Current rec.
481 my($recN);
482 my($recX);
483 my($fincrash); # First record in the crash window.
484 my($lincrash); # Last record in the crash window.
485 my($frecF); # Upper bound when b-searching for first record in crash window.
486 my($lrecF); # Lower bound when b-searching for first record in crash window.
487 my($frecL); # Lower bound when b-searching for last record in crash window.
488 my($lrecL); # Upper bound when b-searching for last record in crash window.
489 my($rows);
490 my(@rowarr);
491 my($rv);
492 my($err);
493
494 $stable = lc($series);
495 $stmnt = "SELECT recnum FROM $stable WHERE recnum = (SELECT min(recnum) FROM $stable)";
496 $rows = $dbh->selectall_arrayref($stmnt, undef);
497
498 # Stupid perl.
499 $err = (NoErr($rows, \$dbh, $stmnt) == 1) ? 0 : 1;
500
501 if (!$err)
502 {
503 @rowarr = @$rows;
504
505 if ($#rowarr < 0)
506 {
507 # No rows in the series table
508 print STDERR "This script was run on a series with no records in it.\n";
509 $err = 1;
510 }
511 else
512 {
513 $min = $rowarr[0]->[0];
514 }
515 }
516
517 if (!$err)
518 {
519 $stmnt = "SELECT recnum FROM $stable WHERE recnum = (SELECT max(recnum) FROM $stable)";
520
521 $rows = $dbh->selectall_arrayref($stmnt, undef);
522 $err = (NoErr($rows, \$dbh, $stmnt) == 1) ? 0 : 1;
523
524 if (!$err)
525 {
526 @rowarr = @$rows;
527
528 if ($#rowarr < 0)
529 {
530 # No rows in the series table
531 print STDERR "This script was run on a series with no records in it.\n";
532 $err = 1;
533 }
534 else
535 {
536 $max = $rowarr[0]->[0];
537 }
538 }
539
540 # We assume that the last record in the series has a session record. If this is not the case, then skip this series.
541 if (!HasSession($dbh, $stable, $max))
542 {
543 $err = 1;
544 }
545 }
546
547 if (!$err)
548 {
549 # Given $min and $max, find the first record in the series that has a drms_session record. We will consider records
550 # before this to not exist. B-search for the first record with a drms_session record. Start with $min.
551 my($tryRec);
552 my($oldTryRec); # From the previous iteration.
553 my($validRec);
554 my($hasSession);
555 my($lbound);
556 my($ubound);
557
558 $frec = undef;
559
560 $tryRec = $min;
561 $lbound = $min;
562 $ubound = $max;
563
564 while (1)
565 {
566 # If $tryRec doesn't change between iterations, then there are no more records to try and we
567 # did not find any records that have a session.
568 if (defined($oldTryRec) && $tryRec == $oldTryRec)
569 {
570 $err = 1;
571 last;
572 }
573
574 $hasSession = HasSession($dbh, $stable, $tryRec);
575
576 if (!defined($hasSession))
577 {
578 $err = 1;
579 last;
580 }
581
582 if ($hasSession)
583 {
584 # If the record immediately below $tryRec does not have a drms_session row, then $tryRec is the first DRMS record
585 # with a drms_session row.
586 $validRec = GetValidRec($dbh, $stable, $tryRec - 1, 1, $lbound);
587
588 if (defined($validRec))
589 {
590 $hasSession = HasSession($dbh, $stable, $validRec);
591
592 if (defined($hasSession))
593 {
594 if (!$hasSession)
595 {
596 $frec = $tryRec;
597 last;
598 }
599 }
600 else
601 {
602 $err = 1;
603 last;
604 }
605 }
606 else
607 {
608 # If there is no record immediately below $tryRec, then $tryRec is the first DRMS record
609 # with a drms_session row.
610 $frec = $tryRec;
611 last;
612 }
613
614 $ubound = $tryRec;
615 # GetValidRec() will find the next EARLIER record (it may not have a drms_session log).
616 $validRec = GetValidRec($dbh, $stable, $tryRec - floor(($ubound - $lbound) / 2), 1, $lbound);
617
618 if (!defined($validRec))
619 {
620 $err = 1;
621 last;
622 }
623 else
624 {
625 $oldTryRec = $tryRec;
626 $tryRec = $validRec;
627 }
628 }
629 else
630 {
631 $lbound = $tryRec;
632
633 # GetValidRec() will find the next LATER record (it may not have a drms_session log).
634 $validRec = GetValidRec($dbh, $stable, $tryRec + floor(($ubound - $lbound) / 2), 0, $ubound);
635
636 if (!defined($validRec))
637 {
638 $err = 1;
639 last;
640 }
641 else
642 {
643 $oldTryRec = $tryRec;
644 $tryRec = $validRec;
645 }
646 }
647 }
648 }
649
650 if (!$err)
651 {
652 $lrec = $max;
653
654 # The first record may actually not be in the before state, but in the in state or in the after state.
655
656 # Because we remove old session records, some records of the series may have no session record. We cannot
657 # use those records to gauge whether or not a range of records is in the crash window. Treat such records
658 # as invalid or missing records. In general, we remove only old session records. If the first record of a series
659 # has no session record, then try a later record (move toward the LAST record in the series.).
660
661 # Starting at the first record in the series ($frec), find the first valid record heading in the
662 # direction of the LAST record in the series ($max). We actually already know that $frec is a valid record.
663 ($locF, $recN) = GetLoc($dbh, $stable, $frec, 0, $frec, $lrec, $bcrash, $ecrash);
664
665 DebugWrite("First valid rec for $stable is $recN (tried $frec). loc is $locF.");
666
667 if ($locF eq "E" || $recN == -1)
668 {
669 # No valid records in the series. This is the same as there being no records created during crash window.
670 $state = "notfound";
671 }
672 else
673 {
674 # Starting at the last record in the series ($lrec), find the first valid record heading in the
675 # direction of the FIRST record in the series ($min).
676 ($locL, $recX) = GetLoc($dbh, $stable, $lrec, 1, $frec, $lrec, $bcrash, $ecrash);
677
678 DebugWrite("Last valid rec for $stable is $recX (tried $lrec). loc is $locL.\n");
679
680 if ($locF eq "I")
681 {
682 if ($locL eq "I")
683 {
684 # Both the min and max of the series are in the crash window.
685 $fincrash = $frec;
686 $lincrash = $lrec;
687 $state = "found";
688 }
689 else
690 {
691 # The first record is in the crash window. This is the first record in the crash window.
692 $state = "in";
693 $fincrash = $recN;
694 $frecL = $recN; # current rec in crash window; lower bound for b-search.
695 $lrecL = $lrec; # upper bound for b-search.
696 }
697 }
698 else
699 {
700 if ($locL eq "I")
701 {
702 # The last record is in the crash window. This is the last record in the crash window.
703 $state = "in";
704 $lincrash = $recX;
705 $frecF = $recX; # current rec in crash window; upper bound for b-search.
706 $lrecF = $frec; # lower bound for b-search.
707 }
708 else
709 {
710 # Neither min nor max is in the crash window.
711 if ($locF eq "B")
712 {
713 $state = "before";
714 $rec = $frec;
715 }
716 else
717 {
718 # frec is after the crash window - NOTHING TO RETURN (the series is completely
719 # outside the crash window - it is after the crash window).
720 $state = "notfound";
721 }
722 }
723 }
724 }
725
726 if (!$err)
727 {
728
729 DebugWrite("Starting FindRecs() loop in state $state.");
730 while(1)
731 {
732 if ($state eq "found" || $state eq "notfound")
733 {
734 last;
735 }
736 elsif ($state eq "before")
737 {
738 $frec = $rec;
739 $rec = $frec + ($lrec - $frec) / 2;
740
741 ($loc, $rec) = GetLoc($dbh, $stable, $rec, 1, $frec, $lrec, $bcrash, $ecrash);
742 DebugWrite("rec is $rec, loc is $loc.");
743 if ($rec == $frec)
744 {
745 # Was not able to find a NEW record with a record number ge to the original
746 # recnum. Try finding a new record by searching in the opposite direction
747 # (toward $max).
748 $rec = $frec + ($lrec - $frec) / 2;
749 ($loc, $rec) = GetLoc($dbh, $stable, $rec, 0, $frec, $lrec, $bcrash, $ecrash);
750 DebugWrite("rec is $rec, loc is $loc.");
751 }
752
753 if ($rec == $lrec)
754 {
755 # No records between $frec and $lrec, and we know the $frec is before
756 # the crash window, and $lrec is after. So, there are no records in the crash window.
757 $state = "notfound";
758 }
759 else
760 {
761 if ($loc eq "B")
762 {
763 $state = "before";
764 }
765 elsif ($loc eq "A")
766 {
767 $state = "after";
768 }
769 else
770 {
771 # $frec is before the crash window, $lrec is after the crash window, and $rec
772 # is in the crash window.
773 $state = "in";
774 $frecF = $rec;
775 $lrecF = $frec;
776 $frecL = $rec;
777 $lrecL = $lrec;
778 }
779 }
780 }
781 elsif ($state eq "in")
782 {
783 if (!defined($fincrash))
784 {
785 # $frecF is inside crash window, $lrecF is after crash window (or last record in
786 # crash window).
787
788 DebugWrite("Finding first in crash window: rec $frecF is in crash, rec $lrecF is before crash.");
789 $fincrash = FindFirstRec($dbh, $stable, $frec, $lrec, $bcrash, $ecrash, $frecF, $lrecF);
790 }
791
792 if (!defined($lincrash))
793 {
794 # $recL is inside crash window, $frecL is before crash window (or first record
795 # in crash window).
796 DebugWrite("Finding last in crash window: rec $frecL is in crash, rec $lrecL is before crash.");
797 $lincrash = FindLastRec($dbh, $stable, $frec, $lrec, $bcrash, $ecrash, $frecL, $lrecL);
798 }
799
800 if (defined($fincrash) && defined($lincrash))
801 {
802 $state = "found";
803 }
804 }
805 elsif ($state eq "after")
806 {
807 $lrec = $rec;
808 $rec = $frec + ($lrec - $frec) / 2;
809
810 ($loc, $rec) = GetLoc($dbh, $stable, $rec, 0, $frec, $lrec, $bcrash, $ecrash);
811 DebugWrite("rec is $rec, loc is $loc.");
812 if ($rec == $lrec)
813 {
814 # Was not able to find a NEW record with a record number le to the original
815 # recnum. Try finding a new record by searching in the opposite direction
816 # (toward $min).
817 $rec = $frec + ($lrec - $frec) / 2;
818 ($loc, $rec) = GetLoc($dbh, $stable, $rec, 1, $frec, $lrec, $bcrash, $ecrash);
819 DebugWrite("rec is $rec, loc is $loc.");
820 }
821
822 if ($rec == $frec)
823 {
824 # No records between $frec and $lrec, and we know the $frec is before
825 # the crash window, and $lrec is after. So, there are no records in the crash window.
826 $state = "notfound";
827 }
828 else
829 {
830 if ($loc eq "B")
831 {
832 $state = "before";
833 }
834 elsif ($loc eq "A")
835 {
836 $state = "after";
837 }
838 else
839 {
840 # $frec is before the crash window, $lrec is after the crash window, and $rec
841 # is in the crash window.
842 $state = "in";
843 $frecF = $rec;
844 $lrecF = $frec;
845 $frecL = $rec;
846 $lrecL = $lrec;
847 }
848 }
849 }
850 } # while loop
851 }
852 }
853
854 if (!$err && $state eq "found")
855 {
856 # We have the first record in the crash window, and the last. Now we need to get all the records between those two record,
857 # including those two records, from the cache.
858
859 DebugWrite("First rec in crash $fincrash, last rec in crash $lincrash.");
860
861 # First cache the records (some may not be cached).
862 CacheRecs($dbh, $series, $fincrash, $lincrash);
863
864 # Then fetch them.
865 $rv = GetCachedRecs($series, $fincrash, $lincrash);
866 }
867
868 return $rv
869 }
870
871 # Given a record number as input, determine whether that record could have been created
872 # within the crash window. The record number provided as input might not be a valid
873 # record number since its value may be the result of an arithmetic calcuation. This function
874 # will either repeatedly subtract one from, and add one to, the provided record number, until
875 # either a valid record number is retrieved, or the min/max record number is reached (the
876 # min and max record numbers are provided as input).
877
878 # This function returns an array whose first element is the determined location code,
879 # whose second element is a valid record number. The location codes include: "E" - error,
880 # "B" - the record specified was created before the crash window, "I" - the record
881 # specified was created during the crash window, "A" - the record specified was created
882 # after the crash window.
883
884 # The record-finding algorithm is currently not efficient - there is a linear search for
885 # the next valid record number. A b-search could be performed instead. The way this would work
886 # is that you'd do a query to see if there are ANY records between $recno and $min
887 # (if $down == 1), and if so, cut this window in to two equal-size subwindows. Then check the
888 # window closest to $recno. If there are records there, then cut that window in half. If there are
889 # no records in the top-level half window between $recno and ($recno - $min) / 2, then check
890 # the half window between ($recno - $min) / 2 and $min. Keep cutting in half till you get
891 # a window with one record in it.
892 sub GetLoc
893 {
894 my($dbh,
895 $stable,
896 $recno,
897 $down,
898 $min, # The first record in the series - can't go below this.
899 $max, # The last record in the series - can't go above this.
900 $bcrash,
901 $ecrash) = @_;
902 my($err);
903 my($stmnt);
904 my($starttime);
905 my($endtime);
906 my($rows);
907 my(@rowarr);
908 my($wincls);
909 my($winfar);
910 my($winhaf);
911 my(@win);
912 my(@rv);
913
914 # $rec might not be a valid recnum - there was an arithmetic manipulation performed on it below.
915 # So, make it a valid recnum. If $state is before, then round down, then start subtracting 1
916 # till we find a valid recnum. If $state is after, then round up and start adding 1 till we
917 # find a valid recnum. If $state is in, then subtract 1 from $frec till we find a valid recnum, and
918 # add 1 to $lrec till we find a valid recnum.
919
920 # Initialize return value.
921 @rv = ("E", -1);
922
923 # First, get a valid record, and fetch its sessionid.
924 if ($down)
925 {
926 $recno = floor($recno);
927 }
928 else
929 {
930 $recno = ceil($recno);
931 }
932
933 if ($recno < $min)
934 {
935 $recno = $min;
936 }
937
938 if ($recno > $max)
939 {
940 $recno = $max;
941 }
942
943 $err = 0;
944
945 # First, check to see if $recno is already a valid record. If so, we can avoid b-searching through a lot of
946 # records.
947 @win = CheckWindow($dbh, $stable, $recno, $recno);
948
949 if ($#win < 0)
950 {
951 # Create the initial b-search windows.
952 if ($down)
953 {
954 $winfar = $min;
955 $wincls = $recno;
956 $winhaf = floor($min + ($recno - $min) / 2);
957 }
958 else
959 {
960 $wincls = $recno;
961 $winfar = $max;
962 $winhaf = ceil($recno + ($max - $recno) / 2);
963 }
964
965 while (1)
966 {
967 # Check close window
968 if ($down)
969 {
970 @win = CheckWindow($dbh, $stable, $winhaf, $wincls);
971 }
972 else
973 {
974 @win = CheckWindow($dbh, $stable, $wincls, $winhaf);
975 }
976
977 if ($#win < 0)
978 {
979 # Check far window
980 if ($down)
981 {
982 @win = CheckWindow($dbh, $stable, $winfar, $winhaf -1);
983 }
984 else
985 {
986 my($add) = $winhaf + 1;
987 @win = CheckWindow($dbh, $stable, $winhaf + 1, $winfar);
988 }
989
990 # in case we need to cut this window in half
991 $wincls = $winhaf;
992 }
993 else
994 {
995 $winfar = $winhaf;
996 }
997
998 if ($#win < 0)
999 {
1000 # No records outside of $recno - done
1001 @rv = ("E", -1);
1002 $err = 1;
1003 last;
1004 }
1005 elsif ($#win == 0)
1006 {
1007 # One record outside of $recno - done
1008 $recno = $win[0]->[0];
1009 ($starttime, $endtime) = GetTimes($dbh, $stable, $win[0]->[0], $win[0]->[1], $win[0]->[2]);
1010 last;
1011 }
1012 else
1013 {
1014 # More than one record in the window - cut it in half and go again.
1015 # But if there are two adjacent records in the window, then
1016 # make $winhaf = $wincls.
1017 #if ($#win == 1 && abs($win[0]->[0] - $win[0]->[0]) == 1)
1018 if (abs($wincls - $winfar) == 1)
1019 {
1020 $winhaf = $wincls;
1021 }
1022 else
1023 {
1024 if ($down)
1025 {
1026 $winhaf = floor($winfar + ($wincls - $winfar) / 2);
1027 }
1028 else
1029 {
1030 $winhaf = ceil($wincls + ($winfar - $wincls) / 2);
1031 }
1032 }
1033 }
1034 }
1035 }
1036 else
1037 {
1038 # Found $recno in the series. Set $sessionid and $sessionns.
1039 ($starttime, $endtime) = GetTimes($dbh, $stable, $win[0]->[0], $win[0]->[1], $win[0]->[2]);
1040 }
1041
1042 if (!$err)
1043 {
1044 # We have a recno now.
1045 @rv = ("E", $recno);
1046 }
1047
1048 if (!$err)
1049 {
1050 if (defined($starttime) && defined($endtime))
1051 {
1052 my($strp);
1053
1054 $strp = DateTime::Format::Strptime->new(pattern => '%Y-%m-%d %T');
1055
1056 $starttime = $strp->parse_datetime($starttime);
1057 $endtime = $strp->parse_datetime($endtime);
1058 $bcrash = $strp->parse_datetime($bcrash);
1059 $ecrash = $strp->parse_datetime($ecrash);
1060
1061 # The record is IN the crash window if either the endtime of the record's session is in the
1062 # crash window, or the starttime of the record's session is in the crash window.
1063 # one row
1064 DebugWrite("Time comparisons: recno is $recno, start is $starttime, end is $endtime.");
1065 if (($endtime ge $bcrash && $endtime le $ecrash) ||
1066 ($starttime ge $bcrash && $starttime le $ecrash) ||
1067 ($starttime le $bcrash && $endtime ge $ecrash))
1068 {
1069 # Rec was created IN the crash window.
1070 @rv = ("I", $recno);
1071 }
1072 elsif ($endtime lt $bcrash)
1073 {
1074 # Rec was created BEFORE the crash window.
1075 @rv = ("B", $recno);
1076 }
1077 else
1078 {
1079 if ($starttime gt $endtime)
1080 {
1081 # This shouldn't happen!
1082 print STDERR "This cannot conceivably happen: starttime $starttime, endtime $endtime\n";
1083 $err = 1;
1084 }
1085 else
1086 {
1087 # Rec was created AFTER the crash window.
1088 @rv = ("A", $recno);
1089 }
1090 }
1091 }
1092 else
1093 {
1094 # The record probably wasn't created properly. An error of "E" will be returned.
1095 }
1096 }
1097
1098 return @rv;
1099 }
1100
1101 sub CacheRecs
1102 {
1103 my($dbh,
1104 $series,
1105 $firstRec,
1106 $lastRec) = @_;
1107
1108 if ($firstRec <= $lastRec)
1109 {
1110 my($stmnt);
1111 my($rows);
1112 my(@rowarr);
1113
1114 $stmnt = "SELECT recnum, sunum, sessionns, sessionid from $series WHERE recnum >= $firstRec AND recnum <= $lastRec";
1115 $rows = $dbh->selectall_arrayref($stmnt, undef);
1116
1117 if (NoErr($rows, \$dbh, $stmnt))
1118 {
1119 @rowarr = @$rows;
1120 foreach my $row (@rowarr)
1121 {
1122 $gRecCache->{$series}->{$row->[0]} = [$row->[0], $row->[1], $row->[2], $row->[3]];
1123 }
1124 }
1125
1126 # Create records for invalid/missing records in [$firstRec, $lastRec] too.
1127 for (my $iRec = $firstRec; $iRec <= $lastRec; $iRec++)
1128 {
1129 if (!exists($gRecCache->{$series}->{$iRec}))
1130 {
1131 $gRecCache->{$series}->{$iRec} = [];
1132 }
1133 }
1134 }
1135 }
1136
1137 sub PurgeRecs
1138 {
1139 my($series) = @_;
1140
1141 if (exists($gRecCache->{$series}))
1142 {
1143 delete($gRecCache->{$series});
1144 }
1145 }
1146
1147 # Returns a reference to an array of valid series records (series, recnum, sunum) with recnums in [$firstRec, $lastRec]. If any record in [$firstRec, $lastRec]
1148 # is not cached, then an error is returned (an undefined array reference).
1149 sub GetCachedRecs
1150 {
1151 my($series, $firstRec, $lastRec) = @_;
1152
1153 my($rv);
1154 my($rec);
1155
1156 if (exists($gRecCache->{$series}))
1157 {
1158 for (my $iRec = $firstRec; $iRec <= $lastRec; $iRec++)
1159 {
1160 # DebugWrite("Fetching $series:$iRec from cache.");
1161
1162 $rec = $gRecCache->{$series}->{$iRec};
1163 if (scalar(@$rec) > 0)
1164 {
1165 if (!defined($rv))
1166 {
1167 $rv = [];
1168 }
1169
1170 # If scalar(@$rec) == 0, then there is no valid record for that recnum.
1171 push(@$rv, [$series, $rec->[0], $rec->[1]])
1172 }
1173 }
1174 }
1175
1176 return $rv;
1177 }
1178
1179 # Returns (recnum, sessionns, sessionid) tuples for all valid records specified by the range [$beg-$end]. To make space in the db,
1180 # old session records get manually deleted from time to time. But in this function, it is assumed that the recnums provided
1181 # are for records that all have session records.
1182 sub CheckWindow
1183 {
1184 my($dbh,
1185 $stable,
1186 $beg,
1187 $end) = @_;
1188
1189 my($firstRec);
1190 my($lastRec);
1191 my($stmnt);
1192 my($rows);
1193 my(@rowarr);
1194 my($err);
1195 my($subarr);
1196 my(@rv);
1197
1198
1199 # Cache (download from db) all valid records [$beg, $end] that have not been cached.
1200 $firstRec = undef;
1201 $lastRec = undef;
1202
1203 for (my $iRec = $beg; $iRec <= $end; $iRec++)
1204 {
1205 if (exists($gRecCache->{$stable}->{$iRec}))
1206 {
1207 if (defined($firstRec))
1208 {
1209 $lastRec = $iRec - 1; # if $firstRec is defined, then $iRec >= 1
1210 CacheRecs($dbh, $stable, $firstRec, $lastRec);
1211 $firstRec = undef;
1212 $lastRec = undef;
1213 }
1214 }
1215 else
1216 {
1217 if (!defined($firstRec))
1218 {
1219 $firstRec = $iRec;
1220 }
1221 }
1222 }
1223
1224 # Might be one more block of records to cache.
1225 if (defined($firstRec))
1226 {
1227 $lastRec = $end;
1228 CacheRecs($dbh, $stable, $firstRec, $lastRec);
1229 $firstRec = undef;
1230 $lastRec = undef;
1231 }
1232
1233 # All records [$beg, $end] have been cached. For all invalid or missing records I, scalar(@{$gRecCache->{$stable}->{I}}) == 0.
1234 for (my $iRec = $beg; $iRec <= $end; $iRec++)
1235 {
1236 if (scalar(@{$gRecCache->{$stable}->{$iRec}}) != 0)
1237 {
1238 # ACK!!! Cannot push an array into an array! If you try, push() will append each element in the subarray into the
1239 # parent array. Instead, make the subarray a reference, and push the reference into the parent array.
1240 # We are pushing [recnum, sessionns, sessionid] into the returned array.
1241 $subarr = [$gRecCache->{$stable}->{$iRec}->[0], $gRecCache->{$stable}->{$iRec}->[2], $gRecCache->{$stable}->{$iRec}->[3]];
1242 push(@rv, $subarr);
1243 }
1244 }
1245
1246 return @rv;
1247 }
1248
1249 sub FindFirstRec
1250 {
1251 my($dbh,
1252 $stable,
1253 $min,
1254 $max,
1255 $bcrash,
1256 $ecrash,
1257 $recin, # record inside crash window
1258 $recout # record before crash window (or first record in crash window)
1259 ) = @_;
1260
1261 return BSearch($dbh, $stable, $min, $max, $bcrash, $ecrash, $recin, $recout);
1262 }
1263
1264 sub FindLastRec
1265 {
1266 my($dbh,
1267 $stable,
1268 $min,
1269 $max,
1270 $bcrash,
1271 $ecrash,
1272 $recin, # record inside crash window
1273 $recout # record after crash window (or last record in crash window)
1274 ) = @_;
1275
1276 return BSearch($dbh, $stable, $min, $max, $bcrash, $ecrash, $recin, $recout);
1277 }
1278
1279 # returns last record in crash window if $recin < $recout, the first record in the crash
1280 # window if $recin > $recout, and -1 if $recin == $recout.
1281 sub BSearch
1282 {
1283 my($dbh,
1284 $stable,
1285 $min,
1286 $max,
1287 $bcrash,
1288 $ecrash,
1289 $recin,
1290 $recout
1291 ) = @_;
1292
1293 my($rv);
1294 my($down);
1295 my($loc);
1296 my($rec);
1297
1298 if ($recin == $recout)
1299 {
1300 $rv = -1;
1301 }
1302 elsif ($recin < $recout)
1303 {
1304 $down = 0;
1305
1306 ($loc, $recout) = GetLoc($dbh, $stable, $recout, 0, $min, $max, $bcrash, $ecrash);
1307
1308 if ($loc eq "I")
1309 {
1310 $rv = -1; # error
1311 }
1312 else
1313 {
1314 ($loc, $recin) = GetLoc($dbh, $stable, $recin, 0, $min, $max, $bcrash, $ecrash);
1315
1316 if ($loc ne "I")
1317 {
1318 $rv = -1; # error
1319 }
1320 }
1321 }
1322 else
1323 {
1324 $down = 1;
1325
1326 ($loc, $recout) = GetLoc($dbh, $stable, $recout, 1, $min, $max, $bcrash, $ecrash);
1327
1328 if ($loc eq "I")
1329 {
1330 $rv = -1; # error
1331 }
1332 else
1333 {
1334 ($loc, $recin) = GetLoc($dbh, $stable, $recin, 1, $min, $max, $bcrash, $ecrash);
1335
1336 if ($loc ne "I")
1337 {
1338 $rv = -1; # error
1339 }
1340 }
1341 }
1342
1343 while (!defined($rv))
1344 {
1345 if ($down)
1346 {
1347 $rec = $recin; # save current rec
1348 $recin = $recout + ($recin - $recout) / 2; # new rec
1349
1350 ($loc, $recin) = GetLoc($dbh, $stable, $recin, 1, $min, $max, $bcrash, $ecrash);
1351
1352 if ($recin == $recout)
1353 {
1354 # We were not able to find a NEW record between $recin and $recout. Try searching up
1355 # from $recin toward $rec (the original $recin).
1356 $recin = $recout + ($rec - $recout) / 2; # new rec
1357 ($loc, $recin) = GetLoc($dbh, $stable, $recin, 0, $min, $max, $bcrash, $ecrash);
1358
1359 if ($recin == $rec)
1360 {
1361 # No records between $recin and $rec, so $rec is the last record in the crash window.
1362 $rv = $rec;
1363 }
1364 }
1365
1366 if (!defined($rv))
1367 {
1368 # We found a record between $recin and $recout, check its location.
1369 if ($loc eq "B")
1370 {
1371 $recout = $recin;
1372 $recin = $rec;
1373 }
1374 elsif ($loc ne "I")
1375 {
1376 # error
1377 print STDERR "Something went wrong 1.\n";
1378 $rv = -1;
1379 }
1380
1381 # If $recin is inside the crash window, then we stay in this loop, using the new $recin as the
1382 # upper bound for the next iteration of b-search.
1383 }
1384 }
1385 else
1386 {
1387 $rec = $recin; # save current rec
1388 $recin = $recin + ($recout - $recin) / 2; # new rec
1389
1390 ($loc, $recin) = GetLoc($dbh, $stable, $recin, 0, $min, $max, $bcrash, $ecrash);
1391
1392 if ($recin == $recout)
1393 {
1394 # We were not able to find a NEW record between $recin and $recout. Try searching down
1395 # from $recin toward $rec (the original $recin).
1396 $recin = $rec + ($recout - $rec) / 2; # new rec
1397 ($loc, $recin) = GetLoc($dbh, $stable, $recin, 1, $min, $max, $bcrash, $ecrash);
1398
1399 if ($recin == $rec)
1400 {
1401 # No records between $recin and $rec, so $rec is the last record in the crash window.
1402 $rv = $rec;
1403 }
1404 }
1405
1406 if (!defined($rv))
1407 {
1408 if ($loc eq "A")
1409 {
1410 $recout = $recin;
1411 $recin = $rec;
1412 }
1413 elsif ($loc ne "I")
1414 {
1415 # error
1416 print STDERR "Something went wrong 2.\n";
1417 $rv = -1;
1418 }
1419
1420 # If $recin is inside the crash window, then we stay in this loop, using the new $recin as the
1421 # lower bound for the next iteration of b-search.
1422 }
1423 }
1424 }
1425
1426
1427 return $rv;
1428 }
1429
1430 sub DebugWrite
1431 {
1432 my($msg) = @_;
1433
1434 if (&DEBUG_ON)
1435 {
1436 print STDERR $msg . "\n";
1437 }
1438 }