| 1 |
#! /bin/csh -f |
| 2 |
# general script for migrating datasets from DRMS stored DSDS datasets |
| 3 |
# into DRMS per record datasets using ingestdsds_a. |
| 4 |
|
| 5 |
# Works with workflow systeem |
| 6 |
|
| 7 |
set echo |
| 8 |
set noglob |
| 9 |
|
| 10 |
# Set the following lines for the target series to migrate |
| 11 |
set SERIES = fd_M_96m_01d |
| 12 |
# DRMS series of datasets |
| 13 |
set SUM_SER = "dsds.mdi__lev1_8__fd_M_96m_01d" |
| 14 |
# new DRMS series of records |
| 15 |
set NEW_SER = mdi.fd_M_96m_lev18 |
| 16 |
# control file for ingest_dsds_a |
| 17 |
set INGESTMAP = /home/phil/jsoc/proj/myproj/apps/fd_M_96m/fd_M_96m.map |
| 18 |
|
| 19 |
set TEMP = temp.DIRS |
| 20 |
|
| 21 |
set WANTLOW = `cat wantlow` |
| 22 |
set WANTLOW_t = `time_convert time=$WANTLOW` |
| 23 |
set WANTLOW = `time_convert zone=TAI s=$WANTLOW_t` |
| 24 |
|
| 25 |
set WANTHIGH = `cat wanthigh` |
| 26 |
set WANTHIGH_t = `time_convert time=$WANTHIGH` |
| 27 |
set WANTHIGH = `time_convert zone=TAI s=$WANTHIGH_t` |
| 28 |
|
| 29 |
set HERE = $cwd |
| 30 |
set babble = $HERE/babble |
| 31 |
mkdir $HERE/qsubs_running |
| 32 |
mkdir $HERE/$TEMP |
| 33 |
set LOG = $HERE/runlog |
| 34 |
|
| 35 |
# wait here if qsub queue too big already |
| 36 |
#while (`qstat -u $USER | wc -l` > 40) |
| 37 |
# sleep 10 |
| 38 |
# echo -n '.' >>$babble |
| 39 |
#end |
| 40 |
|
| 41 |
set FDS = `time_index -d time=$WANTLOW` |
| 42 |
set LDS = `time_index -d time=$WANTHIGH` |
| 43 |
|
| 44 |
# This block of code should be same in each 96-minute-by-day update script |
| 45 |
# the workflow ticket may have a "special" line in which case the FILLGAPSLOW |
| 46 |
# time may be present. If so all gaps between FILLGAPSLOW and wantlow should |
| 47 |
# be plugged with missing records. |
| 48 |
|
| 49 |
set SPECIAL = `grep SPECIAL ticket` |
| 50 |
if ($? == 0) then |
| 51 |
foreach SPECIALWORD ($SPECIAL) |
| 52 |
set $SPECIALWORD |
| 53 |
end |
| 54 |
if ($?FILLGAPSLOW) then |
| 55 |
set FILLGAPSLOWDS = `time_index -d time=$FILLGAPSLOW` |
| 56 |
@ FILLGAPSHIGHDS = $FDS - 1 |
| 57 |
if ($FILLGAPSHIGHDS >= $FILLGAPSLOWDS) then |
| 58 |
# some older gaps should be plugged with missing |
| 59 |
set FILLGAPSHIGH_m = `time_index -m hour=$FILLGAPSHIGHDS` |
| 60 |
@ FILLGAPSHIGH_m = $FILLGAPSHIGH_m + 1440 - 96 |
| 61 |
set FILLGAPSHIGH = `time_index -t minute=$FILLGAPSHIGH_m` |
| 62 |
# set_gaps_missing ds=$NEW_SER low=$FILLGAPSLOW high=$FILLGAPSHIGH > $HERE/$TEMP/gapsfilled |
| 63 |
echo set_gaps_missing ds=$NEW_SER low=$FILLGAPSLOW high=$FILLGAPSHIGH |
| 64 |
endif |
| 65 |
endif |
| 66 |
|
| 67 |
set INSTANCE = $FDS'_'$LDS |
| 68 |
|
| 69 |
set TEMPCMD = $TEMP/CMD.$INSTANCE |
| 70 |
set TEMPLOG = $TEMP/LOG.$INSTANCE |
| 71 |
set TEMPLIST = $TEMP/LIST.$INSTANCE |
| 72 |
rm -f $TEMPCMD $TEMPLOG $TEMPLIST |
| 73 |
|
| 74 |
show_info -pq $SUM_SER"["$FDS-$LDS"]" >$TEMPLIST |
| 75 |
|
| 76 |
set retstatus=0 |
| 77 |
set n = `wc -l < $TEMPLIST` |
| 78 |
if ($n == 0) then |
| 79 |
echo No datasets found for $FDS through $LDS >>$babble |
| 80 |
else |
| 81 |
echo $n hours in $FDS >>$babble |
| 82 |
|
| 83 |
echo "#! /bin/csh -f " >$TEMPCMD |
| 84 |
echo "cd $HERE" >>$TEMPCMD |
| 85 |
echo "hostname >>&$TEMPLOG" >>$TEMPCMD |
| 86 |
echo 'ingest_dsds_a -v -L in=@'"$TEMPLIST out=$NEW_SER map=$INGESTMAP >>&$TEMPLOG" >>$TEMPCMD |
| 87 |
echo 'echo $status >retstatus' >>$TEMPCMD |
| 88 |
echo "rm -f $HERE/qsubs_running/$INSTANCE" >>$TEMPCMD |
| 89 |
|
| 90 |
touch $HERE/qsubs_running/$INSTANCE |
| 91 |
qsub -sync yes -e $HERE/$TEMPLOG -o $HERE/$TEMPLOG -q j.q $HERE/$TEMPCMD |
| 92 |
endif |
| 93 |
|
| 94 |
# while (`/bin/ls $HERE/qsubs_running/ | wc -l` > 0) |
| 95 |
# sleep 10 |
| 96 |
# echo -n '+' >>$babble |
| 97 |
|
| 98 |
echo Done >>$LOG |
| 99 |
if (-e retstatus) set retstatus = `cat retstatus` |
| 100 |
|
| 101 |
exit $retstatus |
