| 1 |
#! /bin/csh -f |
| 2 |
# general script for migrating datasets from DRMS stored DSDS datasets |
| 3 |
# into DRMS per record datasets using ingestdsds_a. |
| 4 |
# start and stop times, FDAY and LDAY, may be set on command line |
| 5 |
# as dates or in this code. |
| 6 |
|
| 7 |
# with no dates given, will update from last day in new series |
| 8 |
|
| 9 |
# set echo |
| 10 |
set noglob |
| 11 |
|
| 12 |
# Set the following lines for the target series to migrate |
| 13 |
# DRMS series of datasets |
| 14 |
set SUM_SER = dsds.mdi__lev1_8__vw_V_06h |
| 15 |
# new DRMS series of records |
| 16 |
set NEW_SER = mdi.vw_V_lev18 |
| 17 |
# control file for ingest_dsds_a |
| 18 |
set INGESTMAP = /home/phil/jsoc/proj/myproj/apps/vw_V_06h/vw_V_lev18.map |
| 19 |
# local directory for working files |
| 20 |
set TEMP = temp.DIRS |
| 21 |
|
| 22 |
set HERE = $cwd |
| 23 |
set babble = $HERE/babble |
| 24 |
mkdir $HERE/qsubs_running |
| 25 |
mkdir $HERE/$TEMP |
| 26 |
set LOG = $HERE/runlog |
| 27 |
|
| 28 |
# wait here if qsub queue too big already |
| 29 |
#while (`qstat -u $USER | wc -l` > 40) |
| 30 |
# sleep 10 |
| 31 |
# echo -n '.' >>$babble |
| 32 |
#end |
| 33 |
|
| 34 |
set WANTLOW = `cat wantlow` |
| 35 |
set WANTLOW_t = `time_convert time=$WANTLOW` |
| 36 |
set WANTLOW = `time_convert zone=TAI s=$WANTLOW_t` |
| 37 |
|
| 38 |
set WANTHIGH = `cat wanthigh` |
| 39 |
set WANTHIGH_t = `time_convert time=$WANTHIGH` |
| 40 |
set WANTHIGH = `time_convert zone=TAI s=$WANTHIGH_t` |
| 41 |
|
| 42 |
set FDS = `time_index -6 time=$WANTLOW` |
| 43 |
set LDS = `time_index -6 time=$WANTHIGH` |
| 44 |
|
| 45 |
# This block of code should be same in each minute-by-hour update script |
| 46 |
# the workflow ticket may have a "special" line in which case the FILLGAPSLOW |
| 47 |
# time may be present. If so all gaps between FILLGAPSLOW and wantlow should |
| 48 |
# be plugged with missing records. |
| 49 |
|
| 50 |
set SPECIAL = `grep SPECIAL ticket` |
| 51 |
if ($? == 0) then |
| 52 |
foreach SPECIALWORD ($SPECIAL) |
| 53 |
set $SPECIALWORD |
| 54 |
end |
| 55 |
if ($?FILLGAPSLOW) then |
| 56 |
set FILLGAPSLOWDS = `time_index -h time=$FILLGAPSLOW` |
| 57 |
@ FILLGAPSHIGHDS = $FDS - 1 |
| 58 |
if ($FILLGAPSHIGHDS >= $FILLGAPSLOWDS) then |
| 59 |
# some older gaps should be plugged with missing |
| 60 |
set FILLGAPSHIGH_m = `time_index -m hour=$FILLGAPSHIGHDS` |
| 61 |
@ FILLGAPSHIGH_m = $FILLGAPSHIGH_m + 59 |
| 62 |
set FILLGAPSHIGH = `time_index -t minute=$FILLGAPSHIGH_m` |
| 63 |
# set_gaps_missing ds=$NEW_SER low=$FILLGAPSLOW high=$FILLGAPSHIGH > $HERE/$TEMP/gapsfilled |
| 64 |
echo set_gaps_missing ds=$NEW_SER low=$FILLGAPSLOW high=$FILLGAPSHIGH |
| 65 |
endif |
| 66 |
endif |
| 67 |
|
| 68 |
# now for making new data |
| 69 |
set INSTANCE = $FDS'_'$LDS |
| 70 |
|
| 71 |
set TEMPCMD = $TEMP/CMD.$INSTANCE |
| 72 |
set TEMPLOG = $TEMP/LOG.$INSTANCE |
| 73 |
set TEMPLIST = $TEMP/LIST.$INSTANCE |
| 74 |
rm -f $TEMPCMD $TEMPLOG $TEMPLIST |
| 75 |
|
| 76 |
show_info -pq $SUM_SER"["$FDS-$LDS"]" >$TEMPLIST |
| 77 |
|
| 78 |
set n = `wc -l < $TEMPLIST` |
| 79 |
set retstatus=0 |
| 80 |
if ($n == 0) then |
| 81 |
echo No datasets found for $FDS through $LDS >>$babble |
| 82 |
else |
| 83 |
echo $n blocks in $FDS >>$babble |
| 84 |
|
| 85 |
echo "#! /bin/csh -f " >$TEMPCMD |
| 86 |
echo "cd $HERE" >>$TEMPCMD |
| 87 |
echo "hostname >>&$TEMPLOG" >>$TEMPCMD |
| 88 |
echo 'ingest_dsds_a -v -L in=@'"$TEMPLIST out=$NEW_SER map=$INGESTMAP >>&$TEMPLOG" >>$TEMPCMD |
| 89 |
# echo '/home/jsoc/cvs/Development/JSOC/bin/linux_x86_64/ingest_dsds_a -v -L in=@'"$TEMPLIST out=$NEW_SER map=$INGESTMAP >>&$TEMPLOG" >>$TEMPCMD |
| 90 |
echo 'echo $status >retstatus' >>$TEMPCMD |
| 91 |
echo "rm -f $HERE/qsubs_running/$INSTANCE" >>$TEMPCMD |
| 92 |
|
| 93 |
touch $HERE/qsubs_running/$INSTANCE |
| 94 |
# qsub -e $HERE/$TEMPLOG -o $HERE/$TEMPLOG -q j.q $HERE/$TEMPCMD |
| 95 |
qsub -sync yes -e $HERE/$TEMPLOG -o $HERE/$TEMPLOG -q j.q $HERE/$TEMPCMD |
| 96 |
endif |
| 97 |
|
| 98 |
# while (`/bin/ls $HERE/qsubs_running/ | wc -l` > 0) |
| 99 |
# sleep 10 |
| 100 |
# echo -n '+' >>$babble |
| 101 |
# end |
| 102 |
|
| 103 |
if (-e retstatus) set retstatus = `cat retstatus` |
| 104 |
|
| 105 |
echo Done $0 $* >>$LOG |
| 106 |
exit $retstatus |
| 107 |
|
