| 1 |
|
#!/home/jsoc/bin/linux_x86_64/activepython |
| 2 |
|
|
| 3 |
+ |
# When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS. |
| 4 |
+ |
|
| 5 |
|
import sys |
| 6 |
|
import getopt |
| 7 |
|
import re |
| 8 |
|
import os |
| 9 |
|
import stat |
| 10 |
+ |
import filecmp |
| 11 |
|
import xml.etree.ElementTree as ET |
| 12 |
|
from subprocess import check_output, CalledProcessError |
| 13 |
|
|
| 74 |
|
} |
| 75 |
|
1;""" |
| 76 |
|
|
| 77 |
+ |
SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script. |
| 78 |
+ |
# It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the |
| 79 |
+ |
# parameter values in this configuration file, modify config.local, then re-run configure. This configuration |
| 80 |
+ |
# file will be updated only if parameters affecting it are modified. If such changes are made to config.local, |
| 81 |
+ |
# please make sure that the sum_rm service is disabled while configure in running. |
| 82 |
+ |
""" |
| 83 |
+ |
|
| 84 |
+ |
SUMRM_DOC = """# sum_rm removes end-of-life SUMS data files to prevent disk-partitions from becomming 100% full. |
| 85 |
+ |
# sum_svc, the main SUMS service, starts this daemon when it is launched. Within an infinite loop, sum_rm sleeps |
| 86 |
+ |
# for SLEEP number of seconds (defined below). When it awakes, it loads the sum_rm configuration file. For each SU, |
| 87 |
+ |
# it then examines the 'effective_date' column within the sum_partn_alloc table of the SUMS database. It does so by |
| 88 |
+ |
# sorting all SUs by effective date (this date is actually an expiration date - the SU is good until the end of that day), |
| 89 |
+ |
# and then, starting with the SU with the oldest effective date, it deletes the SUs. It continues deleting SUs until one |
| 90 |
+ |
# of three conditions is met: |
| 91 |
+ |
# |
| 92 |
+ |
# (a) The disk has at least PART_PERCENT_FREE percent free space. |
| 93 |
+ |
# (b) All SUs have effective_date values in the future. |
| 94 |
+ |
# (c) At least 600 SUs have been deleted (this is defined in the LIMIT statement in the file SUMLIB_RmDoX.pgc). |
| 95 |
+ |
# |
| 96 |
+ |
# After sum_rm stops deleteing SUs, it then sleeps for SLEEP seconds, completing the first iteration of the |
| 97 |
+ |
# infinite loop. |
| 98 |
+ |
""" |
| 99 |
+ |
|
| 100 |
+ |
SUMRM_PARTN_PERCENT_FREE = """ |
| 101 |
+ |
# This is the percentage at which all disk partitions are to be kept free. |
| 102 |
+ |
# If not specified, this defaults to 3. For example, setting PART_PERCENT_FREE = 5 will allow all partitions to |
| 103 |
+ |
# fill to 95% full. Dividing the number of unused blocks by the total number of blocks, and rounding up, |
| 104 |
+ |
# will result in the number specified by PART_PERCENT_FREE. |
| 105 |
+ |
# |
| 106 |
+ |
# NOTE : This behavior was previously controlled by the MAX_FREE_{n} family of parameters. {n} referred to the |
| 107 |
+ |
# disk-partition number and the value of parameter MAX_FREE_{n} was the MINIMUM number of free MB in the |
| 108 |
+ |
# partition [No clue why the word MAX was used]. As of NetDRMS 7.0, MAX_FREE_{n} has no effect.""" |
| 109 |
+ |
|
| 110 |
+ |
SUMRM_SLEEP = """ |
| 111 |
+ |
# The value is the number of seconds to sleep between iterations of the main loop in sum_rm.""" |
| 112 |
+ |
|
| 113 |
+ |
SUMRM_LOG = """ |
| 114 |
+ |
# The value is the log file (opened only at sum_rm startup; the sum_rm pid is appended to this file name).""" |
| 115 |
+ |
|
| 116 |
+ |
SUMRM_MAIL = """ |
| 117 |
+ |
# The value is the email address of the recipient to be notified in case of a problem.""" |
| 118 |
+ |
|
| 119 |
+ |
SUMRM_NOOP = """ |
| 120 |
+ |
# If the value is set to anything other than 0, then sum_rm is rendered inactive. Otherwise, sum_rm is active.""" |
| 121 |
+ |
|
| 122 |
+ |
SUMRM_USER = """ |
| 123 |
+ |
# The value designates the linux user who is allowed to run sum_rm.""" |
| 124 |
+ |
|
| 125 |
+ |
SUMRM_NORUN = """ |
| 126 |
+ |
# This pair of paramters defines a window of time, during which sum_rm will become torpid - in this |
| 127 |
+ |
# window, sum_rm will not scan for SUs to delete, not will it delete any SUs. Each value is an integer, 0-23, that |
| 128 |
+ |
# represents an hour of the day. For example, if NORUN_START=8 and NORUN_STOP=10, then between 8am and 10am |
| 129 |
+ |
# local time, sum_rm will be dormant. To disable this behavior, set both parameters to the same value.""" |
| 130 |
|
|
| 131 |
|
RULESPREFIX = """# Standard things |
| 132 |
|
sp := $(sp).x |
| 161 |
|
# Read arguments |
| 162 |
|
# d - localization directory |
| 163 |
|
# b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.) |
| 164 |
< |
# m - make file |
| 164 |
> |
# s - configure a NetDRMS server (i.e., SUMS server). Otherwise, do client configuration only. A server |
| 165 |
> |
# configuration will modify something that only the production user has access to. An example is the sum_rm.cfg |
| 166 |
> |
# file. To modify that file, the user running configure must be running configure on the SUMS-server host, and |
| 167 |
> |
# must have write permission on sum_rm.cfg. |
| 168 |
|
def GetArgs(args): |
| 169 |
|
rv = bool(0) |
| 170 |
|
optD = {} |
| 171 |
|
|
| 172 |
|
try: |
| 173 |
< |
opts, remainder = getopt.getopt(args, "hd:b:",["dir=", "base="]) |
| 173 |
> |
opts, remainder = getopt.getopt(args, "hd:b:s",["dir=", "base="]) |
| 174 |
|
except getopt.GetoptError: |
| 175 |
|
print('Usage:') |
| 176 |
|
print('localize.py [-h] -d <localization directory> -b <parameter file base>') |
| 189 |
|
optD['dir'] = matchobj.group(1) |
| 190 |
|
elif opt in ("-b", "--base"): |
| 191 |
|
optD['base'] = arg |
| 192 |
+ |
elif opt in ("-s"): |
| 193 |
+ |
optD['server'] = "" |
| 194 |
|
else: |
| 195 |
|
optD[opt] = arg |
| 196 |
|
|
| 807 |
|
|
| 808 |
|
return rv |
| 809 |
|
|
| 810 |
< |
def configureNet(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base, keymap): |
| 810 |
> |
def generateSumRmCfg(defs): |
| 811 |
> |
rv = bool(0) |
| 812 |
> |
# ACK! Remember that Rick renamed these parameters. The ones in config.local are the aliases - do not use those. |
| 813 |
> |
# Use the ones that those map to (defined in config.local.map). |
| 814 |
> |
cFileTmp = defs['SUMLOG_BASEDIR'] + '/' + '.sum_rm.cfg.tmp' |
| 815 |
> |
cFile = defs['SUMLOG_BASEDIR'] + '/' + 'sum_rm.cfg' |
| 816 |
> |
|
| 817 |
> |
# Write a temporary file sum_rm configuration file. |
| 818 |
> |
try: |
| 819 |
> |
with open(cFileTmp, 'w') as fout: |
| 820 |
> |
# Print comment at the top of the configuration file. |
| 821 |
> |
print(SUMRM_COMMENT, file=fout) |
| 822 |
> |
print(SUMRM_DOC, file=fout) |
| 823 |
> |
print(SUMRM_PARTN_PERCENT_FREE, file=fout) |
| 824 |
> |
if 'SUMRM_PART_PERCENT_FREE' in defs: |
| 825 |
> |
print('PART_PERCENT_FREE=' + defs['SUMRM_PART_PERCENT_FREE'], file=fout) |
| 826 |
> |
else: |
| 827 |
> |
print('PART_PERCENT_FREE=3', file=fout) |
| 828 |
> |
|
| 829 |
> |
print(SUMRM_SLEEP, file=fout) |
| 830 |
> |
if 'SUMRM_SLEEP' in defs: |
| 831 |
> |
print('SLEEP=' + defs['SUMRM_SLEEP'], file=fout) |
| 832 |
> |
else: |
| 833 |
> |
print('SLEEP=300', file=fout) |
| 834 |
> |
|
| 835 |
> |
print(SUMRM_LOG, file=fout) |
| 836 |
> |
if 'SUMRM_LOG' in defs: |
| 837 |
> |
print('LOG=' + defs['SUMRM_LOG'], file=fout) |
| 838 |
> |
else: |
| 839 |
> |
print('LOG=/tmp/sum_rm.log', file=fout) |
| 840 |
> |
|
| 841 |
> |
print(SUMRM_MAIL, file=fout) |
| 842 |
> |
# No default for mail - don't send nothing to nobody unless the operator has asked for notifications. |
| 843 |
> |
if 'SUMRM_MAIL' in defs: |
| 844 |
> |
print('MAIL=' + defs['SUMRM_MAIL'], file=fout) |
| 845 |
> |
else: |
| 846 |
> |
print('# MAIL=president@whitehouse.gov', file=fout) |
| 847 |
> |
|
| 848 |
> |
print(SUMRM_NOOP, file=fout) |
| 849 |
> |
if 'SUMRM_NOOP' in defs: |
| 850 |
> |
print('NOOP=' + defs['SUMRM_NOOP'], file=fout) |
| 851 |
> |
else: |
| 852 |
> |
print('NOOP=0', file=fout) |
| 853 |
> |
|
| 854 |
> |
print(SUMRM_USER, file=fout) |
| 855 |
> |
if 'SUMRM_USER' in defs: |
| 856 |
> |
print('USER=' + defs['SUMRM_USER'], file=fout) |
| 857 |
> |
else: |
| 858 |
> |
print('USER=production', file=fout) |
| 859 |
> |
|
| 860 |
> |
print(SUMRM_NORUN, file=fout) |
| 861 |
> |
# Default norun window is to have no such window. This can be accomplished by simply not providing either argument. |
| 862 |
> |
if 'SUMRM_NORUN_START' in defs or 'SUMRM_NORUN_STOP' in defs: |
| 863 |
> |
if 'SUMRM_NORUN_START' in defs: |
| 864 |
> |
print('NORUN_START=' + defs['SUMRM_NORUN_START'], file=fout) |
| 865 |
> |
else: |
| 866 |
> |
print('NORUN_START=0', file=fout) |
| 867 |
> |
if 'SUMRM_NORUN_STOP' in defs: |
| 868 |
> |
print('NORUN_STOP=' + defs['SUMRM_NORUN_STOP'], file=fout) |
| 869 |
> |
else: |
| 870 |
> |
print('NORUN_STOP=0', file=fout) |
| 871 |
> |
else: |
| 872 |
> |
print('# NORUN_START=0', file=fout) |
| 873 |
> |
print('# NORUN_STOP=0', file=fout) |
| 874 |
> |
|
| 875 |
> |
except OSError: |
| 876 |
> |
print('Unable to open sum_rm temporary configuration file ' + cFileTmp + 'for writing.', file=sys.stderr) |
| 877 |
> |
rv = bool(1) |
| 878 |
> |
|
| 879 |
> |
# If the content of the temporary file differs from the content of the existing configuration file, then overwrite |
| 880 |
> |
# the original file. Otherwise, delete the temporary file |
| 881 |
> |
if not rv: |
| 882 |
> |
try: |
| 883 |
> |
if filecmp.cmp(cFile, cFileTmp): |
| 884 |
> |
# Files identical - delete temporary file |
| 885 |
> |
try: |
| 886 |
> |
os.remove(cFileTmp) |
| 887 |
> |
|
| 888 |
> |
except OSError as exc: |
| 889 |
> |
print('Unable to remove temporary file ' + exc.filename + '.', file=sys.stderr) |
| 890 |
> |
print(exc.strerr, file=sys.stderr) |
| 891 |
> |
else: |
| 892 |
> |
# Replace original with temporary file |
| 893 |
> |
try: |
| 894 |
> |
os.rename(cFileTmp, cFile) |
| 895 |
> |
|
| 896 |
> |
except OSError as exc: |
| 897 |
> |
print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr) |
| 898 |
> |
print(exc.strerr, file=sys.stderr) |
| 899 |
> |
rv = bool(1) |
| 900 |
> |
except OSError as exc: |
| 901 |
> |
# One of the files doesn't exist. |
| 902 |
> |
if exc.filename == cFile: |
| 903 |
> |
# We are ok - there might be no configuration file yet. |
| 904 |
> |
# Replace original with temporary file |
| 905 |
> |
try: |
| 906 |
> |
os.rename(cFileTmp, cFile) |
| 907 |
> |
|
| 908 |
> |
except OSError as exc: |
| 909 |
> |
print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr) |
| 910 |
> |
print(exc.strerr, file=sys.stderr) |
| 911 |
> |
rv = bool(1) |
| 912 |
> |
else: |
| 913 |
> |
# There is a problem with the temp file - bail. |
| 914 |
> |
print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr) |
| 915 |
> |
print(exc.strerr, file=sys.stderr) |
| 916 |
> |
rv = bool(1) |
| 917 |
> |
|
| 918 |
> |
return rv |
| 919 |
> |
|
| 920 |
> |
def configureNet(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base, keymap, createSumRmCfg): |
| 921 |
|
rv = bool(0) |
| 922 |
|
|
| 923 |
|
defs = {} |
| 964 |
|
# Write out the parameter files. |
| 965 |
|
if not rv: |
| 966 |
|
rv = writeParamsFiles(base, cfile, mfile, pfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection) |
| 967 |
+ |
|
| 968 |
|
# Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk). |
| 969 |
|
if not rv: |
| 970 |
|
rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget) |
| 971 |
+ |
|
| 972 |
+ |
# Write out the sum_rm.cfg file. |
| 973 |
+ |
if not rv and createSumRmCfg: |
| 974 |
+ |
rv = generateSumRmCfg(defs) |
| 975 |
|
except IOError as exc: |
| 976 |
|
print(exc.strerror, file=sys.stderr) |
| 977 |
|
print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr) |
| 1042 |
|
# Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk). |
| 1043 |
|
if not rv: |
| 1044 |
|
rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget) |
| 1045 |
+ |
|
| 1046 |
+ |
# At Stanford, skip the creation of the sum_rm configuration file. config.local will still |
| 1047 |
+ |
# have the SUMRM parameters, but they will not be used. |
| 1048 |
|
except IOError as exc: |
| 1049 |
|
print(exc.strerror, file=sys.stderr) |
| 1050 |
|
print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr) |
| 1128 |
|
|
| 1129 |
|
# We also need to set the UID of the SUMS manager. We have the name of the |
| 1130 |
|
# SUMS manager (it is in the configuration file) |
| 1131 |
< |
configureNet(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap) |
| 1131 |
> |
configureNet(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap, 'server' in optD) |
| 1132 |
|
else: |
| 1133 |
|
# A Stanford user can override the parameters in configsdp.txt by copying that file to config.local, |
| 1134 |
|
# and then editing config.local. So, if config.local exists, use that. |
