| 1 |
/home/production/cvs/JSOC/doc/devlev1_flatfield.txt 29Aug2012 |
| 2 |
|
| 3 |
This is the FF associated processing that occurs during the |
| 4 |
creation of the def lev1 data. |
| 5 |
============================================================================ |
| 6 |
|
| 7 |
Cronjob on cl1n001 as user jsocprod: |
| 8 |
30 18 * * * /home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/lev0a2deflev1_FULL_PZT_FSN_cron.pl hmi |
| 9 |
30 20 * * * /home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/lev0a2deflev1_FULL_PZT_FSN_cron.pl aia |
| 10 |
|
| 11 |
============================================================================ |
| 12 |
|
| 13 |
Lets look at the run of: |
| 14 |
/home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/lev0a2deflev1_FULL_PZT_FSN_cron.pl hmi |
| 15 |
|
| 16 |
#This is a cron job run by cl1n001 after the start of a new UT day. |
| 17 |
#For example, after 2010.10.26_17:00:00 PDT is UTC day 2010.300_UTC. |
| 18 |
#It will call lev0a2deflev1_FULL_PZT_FSN.pl with the date equal n-4 from |
| 19 |
#the new current date, e.g. after 2010.10.26_17:00:00 PDT, the call |
| 20 |
#will be: |
| 21 |
# lev0a2deflev1_FULL_PZT_FSN.pl hmi 2010.296_UC |
| 22 |
|
| 23 |
#New 16Feb2011 First do a pzt flat |
| 24 |
if($instru eq "hmi") { |
| 25 |
`/home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/pzt_flat_cron.pl`; |
| 26 |
} |
| 27 |
|
| 28 |
#Now make the def lev1 for our given date |
| 29 |
$cmd = "lev0a2deflev1_FULL_PZT_FSN.pl $instru $ord_date"; |
| 30 |
system($cmd); |
| 31 |
|
| 32 |
============================================================================ |
| 33 |
|
| 34 |
/home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/pzt_flat_cron.pl |
| 35 |
|
| 36 |
#Run Richards pzt_flatfield IDL program. |
| 37 |
$cmd = "cd /home/jsocprod/pztflat; echo \"pzt_flatfield, $pztdate\" | /usr/local/bin/idl 1>> $QSUBDIR/pzt2.$PID.log 2>&1"; |
| 38 |
|
| 39 |
============================================================================ |
| 40 |
|
| 41 |
/home/jsoc/cvs/Development/JSOC/scripts/lev0a2deflev1_FULL_PZT_FSN.pl $instru $ord_date |
| 42 |
|
| 43 |
#Run the build of the hmi.lev1 and fill in any hmi.lev1 gaps: |
| 44 |
#lev1_def_gui_called_PZT_FSN -x hmi 2010.298_UTC |
| 45 |
$cmd = "/home/jsoc/cvs/Development/JSOC/base/sums/scripts/lev1_def_gui_called_PZT_FSN -x $instru $ord_date 1> $statlog 2>&1"; |
| 46 |
|
| 47 |
============================================================================ |
| 48 |
|
| 49 |
lev1_def_gui_called_PZT_FSN -x $instru $ord_date |
| 50 |
|
| 51 |
#These requirements must be met (hmi.lev1_probe DB consulted): |
| 52 |
# |
| 53 |
#1. For the current UTC day# (1-366), there must be a file like so: |
| 54 |
#/dds/soc2pipe/hmi,aia/xday/Xmit_All.198 |
| 55 |
#Optionally, the -o[verride] flag can be given for older data before |
| 56 |
#the Xmit_All.ddd was implemented. |
| 57 |
# |
| 58 |
#2. Do a query for the day of interest like so (note TAI): |
| 59 |
#sdo.hk_dayfile[2010.07.14_00:00:00_TAI][129][moc] |
| 60 |
#where MERGED=1. |
| 61 |
#If this exists, then the sdo.lev0_asd_0004 for this day is complete, |
| 62 |
#and ok to use for the definitive lev1. |
| 63 |
# |
| 64 |
#3. Check sdo.fds_orbit_vectors for a record for the given ord date. |
| 65 |
# |
| 66 |
#4. Check that the flatfield has a flatfield_version >= 1. |
| 67 |
#Check for camera 1&2 for hmi, and for the 23 wave_str for aia. |
| 68 |
# |
| 69 |
#5. Check that sdo.master_pointing record exists for our date. |
| 70 |
# |
| 71 |
#6. Check that hmi,aia.temperature_summary_300s exists for our date. |
| 72 |
# |
| 73 |
#7. Check that the manually set No Go file does not exist: |
| 74 |
#/surge40/jsocprod/lev0/data/NoDefLev1[HMI,AIA] |
| 75 |
|
| 76 |
#Now see if flat field has flatfield_version >= 1 (version ck obsolete) |
| 77 |
for($i=1; $i < 3; $i++) { |
| 78 |
$query = sprintf("%s[? t_start <= \$(%s) and t_stop > \$(%s) and CAMERA=%d ?]", 'hmi.flatfield', $fulldateFF, $fulldateFF, $i); |
| 79 |
|
| 80 |
#print "hmi query= $query\n"; #!!TEMP |
| 81 |
#print "Must put single quote around the above\n"; |
| 82 |
$cmd = "show_info key=date,flatfield_version '$query'"; |
| 83 |
print "$cmd\n"; |
| 84 |
@result = `$cmd`; |
| 85 |
#print "Result of flatfield query for $orddate:\n"; |
| 86 |
print "@result"; |
| 87 |
$x = shift(@result); |
| 88 |
if($x =~ /date\tflatfield_version/) { |
| 89 |
$x = shift(@result); #looks like 2010-07-01T17:28:23Z 1 |
| 90 |
($a, $ffver) = split(/\s+/, $x); |
| 91 |
if($ffver >= 0) { #NOTE: now >= 0 is ok, just find one |
| 92 |
print "*OK flatfield found for CAMERA=$i\n"; |
| 93 |
} else { |
| 94 |
print "**NG flatfield_version not >= 1 for CAMERA=$i\n"; |
| 95 |
$OKff = 0; |
| 96 |
} |
| 97 |
} |
| 98 |
else { |
| 99 |
print "**NG flatfield not found for CAMERA=$i\n"; |
| 100 |
$OKff = 0; |
| 101 |
} |
| 102 |
} |
| 103 |
|
| 104 |
#back up 96 fsn for the way cosmic rays are made by module_flatfield_daily |
| 105 |
#See mail: "Re: patching cosmic ray records" Richard Wachter 11/18/10 12:08 |
| 106 |
$firstfsncosmic = $firstfsn - 96; #new 19Nov2010. Change vrbl 2/28/2011 |
| 107 |
$lastfsncosmic = $lastfsn - 96; #new 19Nov2010. Change vrbl 2/28/2011 |
| 108 |
|
| 109 |
$cmd = "time build_lev1_mgr mode=fsn instru=aia dsin=$DSINAIA dsout=$DSOUTAIA bfsn=$firstfsn efsn=$lastfsn numqsub=8 logfile=/usr/local/logs/lev1/build_lev1_mgr_aia.$utcdate.log"; |
| 110 |
|
| 111 |
|
| 112 |
$gdate = &get_date; |
| 113 |
$statlog = "$LOGDIR/module_flat_cron_48_PZT_FSN_$gdate.log"; |
| 114 |
$cmd = "module_flatfield_daily_cron_48_PZT_FSN.pl $instru $orddate $firstfsncosmic $lastfsncosmic 1> $statlog 2>&1"; |
| 115 |
print "$cmd\n"; |
| 116 |
if($EXECUTE) { |
| 117 |
@ffrun = `$cmd`; |
| 118 |
} |
| 119 |
|
| 120 |
============================================================================ |
| 121 |
|
| 122 |
cvs/Development/JSOC/scripts/module_flatfield_daily_cron_48_PZT_FSN.pl $instru $orddate $firstfsncosmic $lastfsncosmic |
| 123 |
|
| 124 |
#Called after the start of a new UT day to make the hmi.cosmic_rays |
| 125 |
#and rotation flatfield and cosmic_ray_post |
| 126 |
#for the day from the hmi.lev1 data (that is typically made just before |
| 127 |
#this call). |
| 128 |
|
| 129 |
$cmd = "module_flatfield_daily_qsub_48_PZT_FSN.pl $indata $firstfsn $lastfsn $mdate"; |
| 130 |
|
| 131 |
============================================================================ |
| 132 |
|
| 133 |
cvs/Development/JSOC/scripts/module_flatfield_daily_qsub_48_PZT_FSN.pl |
| 134 |
|
| 135 |
#Here are the 48 command that we must run: |
| 136 |
@cmds = ( |
| 137 |
"module_flatfield input_series='$inds' cadence=135 cosmic_rays=1 flatfield=1 fid=10054 camera=1 fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum'", |
| 138 |
"module_flatfield input_series='$inds' cadence=135 cosmic_rays=1 flatfield=1 fid=10055 camera=1 fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum'", |
| 139 |
[etc.] |
| 140 |
|
| 141 |
#Now run the combine program |
| 142 |
$cmd = "/home/jsoc/cvs/Development/JSOC/bin/linux_x86_64/module_flatfield_combine camera=2 input_series='su_production.flatfield_fid' datum='$datum' >& $lfile"; |
| 143 |
|
| 144 |
$cmd = "/home/jsoc/cvs/Development/JSOC/bin/linux_x86_64/module_flatfield_combine camera=1 input_series='su_production.flatfield_fid' datum='$datum' >& $lfile"; |
| 145 |
|
| 146 |
|
| 147 |
#New: 10Jan2011 Do cosmic_ray post processing. This populates hmi.cosmic_rays. |
| 148 |
#See mail richard@sun.stanford.edu 01/04/11 10:40 "cosmic ray series" |
| 149 |
#Here are the 24 command that we must run: |
| 150 |
@postcmds = ( |
| 151 |
"cosmic_ray_post input_series='su_production.cosmic_rays' fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum' camera=1 hour=00", |
| 152 |
"cosmic_ray_post input_series='su_production.cosmic_rays' fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum' camera=1 hour=02", |
| 153 |
[etc.] |
| 154 |
|
