cvs/JSOC/localize.py

#!/usr/bin/env python

# When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS.
import sys
import getopt
import re
import os
import stat
import filecmp
from subprocess import check_output, CalledProcessError
import shlex


# Constants
VERS_FILE = 'jsoc_version.h'
SDP_CFG = 'configsdp.txt'
NET_CFG = 'config.local'
NET_CFGMAP = 'config.local.map'
RET_SUCCESS = 0
RET_NOTDRMS = 1

PREFIX = """# This file was auto-generated by localize.py. Please do not edit it directly (running
# configure will run localize.py, which will then overwrite any edits manually performed).
"""

C_PREFIX = """/* This file was auto-generated by localize.py. Please do not edit it directly (running
 * configure will run localize.py, which will then overwrite any edits manually performed). */
"""

PERL_BINPATH = '#!/usr/bin/perl\n'

PERL_INTIAL = """package drmsparams;

use warnings;
use strict;
"""

PERL_FXNS_A = """sub new
{
    my($clname) = shift;

    my($self) =
    {
        _paramsH => undef
    };

    bless($self, $clname);
    $self->{_paramsH} = {};
    $self->initialize();

    return $self;
}

sub DESTROY
{
    my($self) = shift;
}
"""

PERL_FXNS_B = """sub get
{
    my($self) = shift;
    my($name) = shift;
    my($rv);

    if (exists($self->{_paramsH}->{$name}))
    {
        return $self->{_paramsH}->{$name};
    }
    else
    {
        return undef;
    }
}
1;"""

PY_BINPATH = '#!/usr/bin/env python3\n'

PY_ALL = f"__all__ = [ 'DPError', 'DPMissingParameterError', 'DRMSParams' ]"

PY_ERROR_CLASSES = """
class DPError(Exception):
    def __init__(self):
        self._msg = 'DRMS Parameter Error: '

class DPMissingParameterError(DPError):
    def __init__(self, parameter):
        super().__init__()
        self._msg += 'missing DRMS parameter ' + parameter

    def __str__(self):
        return self._msg
"""

PY_FXNS_A = """
class DRMSParams(object):
    def __init__(self):
        self.params = {}
        self.initialize()

    def __del__(self):
        del self.params

    def initialize(self):
"""

PY_FXNS_B = """    def get(self, name):
        if name in self.params:
            return self.params[name]
        else:
            return None
"""

PY_FXNS_C = """    def get_required(self, name):
        try:
            value = self.params[name]
        except:
            raise DPMissingParameterError(name)

        return value

    def getBool(self, name):
        if name in self.params:
            return bool(self.params[name] == '1')
        else:
            return None

    def __getattr__(self, name):
        # only called if object.__getattribute__(self, name) raises; and if that is true, then we want
        # to look in self.params for it, and set the instance attribute if it does exist in self.params
        if name in self.params:
            attr = self.params[name]
            self.__setattr__(name, attr)
        else:
            attr = None

        return attr

    def __setattr__(self, name, value):
        # call neither __setattr__ nor __getattr__
        try:
            params = object.__getattr__(self, 'params')

            # put into self.params dict, overwriting if necessary
            params[name] = value
        except:
            pass

        # store in instance dict as well
        object.__setattr__(self, name, value)
"""

SH_BINPATH = '#!/bin/bash\n'


SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script.
# It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the
# parameter values in this configuration file, modify config.local, then re-run configure. This configuration
# file will be updated only if parameters affecting it are modified. If such changes are made to config.local,
# please make sure that the sum_rm service is disabled while configure in running.
"""

SUMRM_DOC = """# sum_rm removes end-of-life SUMS data files to prevent disk-partitions from becomming 100% full.
# sum_svc, the main SUMS service, starts this daemon when it is launched. Within an infinite loop, sum_rm sleeps
# for SLEEP number of seconds (defined below). When it awakes, it loads the sum_rm configuration file. For each SU,
# it then examines the 'effective_date' column within the sum_partn_alloc table of the SUMS database. It does so by
# sorting all SUs by effective date (this date is actually an expiration date - the SU is good until the end of that day),
# and then, starting with the SU with the oldest effective date, it deletes the SUs. It continues deleting SUs until one
# of three conditions is met:
#
#   (a) The disk has at least PART_PERCENT_FREE percent free space.
#   (b) All SUs have effective_date values in the future.
#   (c) At least 600 SUs have been deleted (this is defined in the LIMIT statement in the file SUMLIB_RmDoX.pgc).
#
# After sum_rm stops deleteing SUs, it then sleeps for SLEEP seconds, completing the first iteration of the
# infinite loop.
"""

SUMRM_PARTN_PERCENT_FREE = """
# This is the percentage at which all disk partitions are to be kept free.
# If not specified, this defaults to 3. For example, setting PART_PERCENT_FREE = 5 will allow all partitions to
# fill to 95% full. Dividing the number of unused blocks by the total number of blocks, and rounding up,
# will result in the number specified by PART_PERCENT_FREE.
#
# NOTE : This behavior was previously controlled by the MAX_FREE_{n} family of parameters. {n} referred to the
# disk-partition number and the value of parameter MAX_FREE_{n} was the MINIMUM number of free MB in the
# partition [No clue why the word MAX was used]. As of NetDRMS 7.0, MAX_FREE_{n} has no effect."""

SUMRM_SLEEP = """
# The value is the number of seconds to sleep between iterations of the main loop in sum_rm."""

SUMRM_LOG = """
# The value is the log file (opened only at sum_rm startup; the sum_rm pid is appended to this file name)."""

SUMRM_MAIL = """
# The value is the email address of the recipient to be notified in case of a problem."""

SUMRM_NOOP = """
# If the value is set to anything other than 0, then sum_rm is rendered inactive. Otherwise, sum_rm is active."""

SUMRM_USER = """
# The value designates the linux user who is allowed to run sum_rm."""

SUMRM_NORUN = """
# This pair of paramters defines a window of time, during which sum_rm will become torpid - in this
# window, sum_rm will not scan for SUs to delete, not will it delete any SUs. Each value is an integer, 0-23, that
# represents an hour of the day. For example, if NORUN_START=8 and NORUN_STOP=10, then between 8am and 10am
# local time, sum_rm will be dormant. To disable this behavior, set both parameters to the same value."""

RULESPREFIX = """# Standard things
sp              := $(sp).x
dirstack_$(sp)  := $(d)
d               := $(dir)
"""

RULESSUFFIX = """# Standard things
d               := $(dirstack_$(sp))
sp              := $(basename $(sp))
"""

ICC_MAJOR = 9
ICC_MINOR = 0
GCC_MAJOR = 3
GCC_MINOR = 0
IFORT_MAJOR = 9
IFORT_MINOR = 0
GFORT_MAJOR = 4
GFORT_MINOR = 2


# Read arguments
#  d - localization directory
#  b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.)
#  s - configure a NetDRMS server (i.e., SUMS server). Otherwise, do client configuration only. A server
#      configuration will modify something that only the production user has access to. An example is the sum_rm.cfg
#      file. To modify that file, the user running configure must be running configure on the SUMS-server host, and
#      must have write permission on sum_rm.cfg.
def GetArgs(args):
    rv = bool(0)
    optD = {}

    try:
        opts, remainder = getopt.getopt(args, "hd:b:s",["dir=", "base="])
    except getopt.GetoptError:
        print('Usage:')
        print('localize.py [-h] -d <localization directory> -b <parameter file base>')
        rv = bool(1)

    if rv == bool(0):
        for opt, arg in opts:
            if opt == '-h':
                print('localize.py [-h] -d <localization directory> -b <parameter file base>')
            elif opt in ("-d", "--dir"):
                regexp = re.compile(r"(\S+)/?")
                matchobj = regexp.match(arg)
                if matchobj is None:
                    rv = bool(1)
                else:
                    optD['dir'] = matchobj.group(1)
            elif opt in ("-b", "--base"):
                optD['base'] = arg
            elif opt in ("-s"):
                optD['server'] = ""
            else:
                optD[opt] = arg

    return optD

def createMacroStr(key, val, keyColLen, status):
    if keyColLen < len(key):
        status = bool(1)
        return None
    else:
        nsp = keyColLen - len(key)
        spaces = str()
        for isp in range(nsp):
            spaces += ' '
        status = bool(0)
        return '#define ' + key + spaces + val + '\n'

def createPerlConst(key, val, keyColLen, status):
    if keyColLen < len(key):
        status = bool(1)
        return None
    else:
        nsp = keyColLen - len(key)
        spaces = str()
        for isp in range(nsp):
            spaces += ' '
        status = bool(0)
        return 'use constant ' + key + ' => ' + spaces + val + ';\n'

def createPyConst(key, val, keyColLen, status):
    if keyColLen < len(key):
        status = bool(1)
        return None
    else:
        nsp = keyColLen - len(key)
        spaces = str()
        for isp in range(nsp):
            spaces += ' '
        status = bool(0)
        return key + ' = ' + spaces + val + '\n'

def createShConst(key, val, status):
    status = bool(0)
    return key + '=' + val + '\n'


def isSupportedPlat(plat):
    regexp = re.compile(r"\s*(^x86_64|^ia32|^ia64|^avx)", re.IGNORECASE)
    matchobj = regexp.match(plat);

    if not matchobj is None:
        return bool(1);
    else:
        return bool(0);

def processMakeParam(mDefs, key, val, platDict, machDict):
    varMach = None
    regexp = re.compile(r"(\S+):(\S+)")
    matchobj = regexp.match(key)
    if not matchobj is None:
        varName = matchobj.group(1)
        varMach = matchobj.group(2)
    else:
        varName = key

    varValu = val

    if varMach is None:
        mDefs.extend(list('\n' + varName + ' = ' + varValu))
    else:
        if isSupportedPlat(varMach):
            # The guard will compare varValu to $JSOC_MACHINE.
            if not varMach in platDict:
                platDict[varMach] = {}
            platDict[varMach][varName] = varValu
        else:
            # The guard will compare varValu to $MACHINETYPE (this is just the hostname of the machine on which localize.py is running).
            if not varMach in machDict:
                machDict[varMach] = {}
            machDict[varMach][varName] = varValu

def processParam(cfgfile, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section):
    status = 0
    parameter_added = {}
    keyCfgSp  = None
    val = None

    if ''.join(section) == 'defs' or not cfgfile:
        if type(line) is str:
            matchobj = regexp.match(line)

            if not matchobj is None:
                # We have a key-value line
                keyCfgSp = matchobj.group(1)
                val = matchobj.group(2)
                print(f'herer, key={keyCfgSp}, val={val}')
        else:
            keyCfgSp, val = list(line.items())

        if keyCfgSp is not None and val is not None:
            # We have a key-value line
            keyCfgSp = matchobj.group(1)
            val = matchobj.group(2)

            # Must map the indirect name to the actual name
            if keymap:
                # Map to actual name only if the keymap is not empty (which signifies NA).
                if keyCfgSp in keymap:
                    key = keymap[keyCfgSp]
                elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':
                    # Ignore parameters that are not useful and shouldn't have been there in the first place. But
                    # they have been released to the world, so we have to account for them.
                    return None
                elif not cfgfile:
                    # Should not be doing mapping for addenda
                    key = keyCfgSp
                else:
                    raise Exception('badKeyMapKey', keyCfgSp)
            else:
                key = keyCfgSp

            matchobj = regexpQuote.match(key)
            if not matchobj is None:
                quote = matchobj.group(1)
                key = matchobj.group(2)

                # master defs dictionary
                defs[key] = val

                parameter_added[key] = val

                # C header file
                if quote == "q":
                    # Add double-quotes
                    cDefs.extend(list(createMacroStr(key, '"' + val + '"', 40, status)))
                elif quote == "p":
                    # Add parentheses
                    cDefs.extend(list(createMacroStr(key, '(' + val + ')', 40, status)))
                elif quote == "a":
                    # Leave as-is
                    cDefs.extend(list(createMacroStr(key, val, 40, status)))
                else:
                    # Unknown quote type
                    raise Exception('badQuoteQual', key)

                if status:
                    raise Exception('paramNameTooLong', key)

                # Make file - val should never be quoted; just use as is
                mDefsGen.extend(list('\n' + key + ' = ' + val))

                # Perl file - val should ALWAYS be single-quote quoted
                # Save const info to a string
                perlConstSection.extend(list(createPerlConst(key, "'" + val + "'", 40, status)))

                # Python file
                pyConstSection.extend(list(createPyConst(key, "'" + val + "'", 40, status)))

                # Shell source file
                shConstSection.extend(list(createShConst(key, "'" + val + "'", status)))

                if status:
                    raise Exception('paramNameTooLong', key)

                # Save initialization information as a string. Now that we've defined
                # constants (the names of which are the parameter names)
                # we can refer to those in the init section. The key variable holds the
                # name of the constant.
                perlInitSection.extend(list("\n  $self->{_paramsH}->{'" + key + "'} = " + key + ';'))

                # The amount of indenting matters! This is Python.
                pyInitSection.extend(list("        self.params['" + key + "'] = " + key + '\n'))
            else:
                # No quote qualifier
                raise Exception('missingQuoteQual', key)
    elif ''.join(section) == 'make' and cfgfile:
        # Configure the remaining make variables defined in the __MAKE__ section of the configuration file. Third-party
        # library make variables are specified in the __MAKE__ section.
        matchobj = regexp.match(line)
        if not matchobj is None:
            # We have a key-value line
            key = matchobj.group(1)
            val = matchobj.group(2)

            # This information is for making make variables only. We do not need to worry about quoting any values
            defs[key] = val
            processMakeParam(mDefsMake, key, val, platDict, machDict)

    return parameter_added

# We have some extraneous line or a newline - ignore.

def process_project_repos(project_includes):
    print(f'[ process_project_repos ]')
    error = False
    ordered_files = []

    # iterate through all proj subdirectories
    for subdirectory in os.listdir('proj'):
        stripped_subdirectory = subdirectory.strip()
        path = os.path.join('proj', stripped_subdirectory)

        # skip any dir in proj that does not have a Rules.mk file
        if os.path.isfile(os.path.join(path, 'Rules.mk')):
            if os.path.isdir(path):
                ordered_files.append(stripped_subdirectory)

    ordered_files.sort()

    for stripped_subdirectory in ordered_files:
        project_includes.append(f'dir     := $(d)/{stripped_subdirectory}')

    return error

def determineSection(line, regexpStyle, regexpDefs, regexpMake):
    matchobj = regexpStyle.match(line)
    if matchobj:
        return 'style'

    matchobj = regexpDefs.match(line)
    if not matchobj is None:
        return 'defs'

    matchobj = regexpMake.match(line)
    if not matchobj is None:
        return 'make'

    return None

# defs is a dictionary containing all parameters (should they be needed in this script)
def parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, project_includes, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
    error = False

    print(f'addenda is {str(addenda)}')

    # Open required config file (config.local)
    try:
        # Examine each line, looking for key=value pairs.
        regexpStyle = re.compile(r"^__STYLE__")
        regexpDefs = re.compile(r"^__DEFS__")
        regexpMake = re.compile(r"^__MAKE__")
        regexpComm = re.compile(r"^\s*#")
        regexpSp = re.compile(r"^s*$")
        regexpQuote = re.compile(r"^\s*(\w):(.+)")
        regexpCustMkBeg = re.compile(r"^_CUST_")
        regexpCustMkEnd = re.compile(r"^_ENDCUST_")
        regexpDiv = re.compile(r"^__")
        regexp = re.compile(r"^\s*(\S+)\s+(\S.*)")

        ignoreKeymap = False
        platDict = {}
        machDict = {}

        # Process the parameters in the configuration file
        if not fin is None:
            for line in fin:
                matchobj = regexpComm.match(line)
                if not matchobj is None:
                    # Skip comment line
                    continue

                matchobj = regexpSp.match(line)
                if not matchobj is None:
                    # Skip whitespace line
                    continue

                newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake)
                if not newSection is None:
                    section = newSection

                if not section:
                    raise Exception('invalidConfigFile', 'line ' + line.strip() + ' is not in any section')

                if section == 'style':
                    # if the config.local file has new in the __STYLE__ section, then ignore the keymap and treat config.local like configsdp.txt;
                    # do not map from NetDRMS config.local parameter names to configsdp.txt names
                    for line in fin:
                        matchobj = regexpDiv.match(line)
                        if matchobj:
                            break;
                        if line.strip(' \n').lower() == 'new' and keymap:
                            ignoreKeymap = True

                    newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake)
                    if not newSection is None:
                        section = newSection
                    continue
                elif section == 'make':

                    # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.
                    # The blocks are defined by _CUST_/_ENDCUST_ tags.
                    matchobj = regexpCustMkBeg.match(line)

                    if not matchobj is None:
                        mDefsMake.extend(list('\n'))
                        for line in fin:
                            matchobj = regexpCustMkEnd.match(line)
                            if not matchobj is None:
                                break;
                            mDefsMake.extend(list(line))
                        newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake)
                        if not newSection is None:
                            section = newSection
                        continue
                    # Intentional fall through to next if statement
                if section == 'defs' or section == 'make':
                    iscfg = bool(1)
                    if ignoreKeymap:
                        keymapActual = None
                    else:
                        keymapActual = keymap
                    ppRet = processParam(iscfg, line, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section)
                else:
                    # Unknown section
                    raise Exception('unknownSection', section)
    except Exception as exc:
        if len(exc.args) >= 2:
            msg = exc.args[0]
        else:
            # re-raise the exception
            raise

        if msg == 'invalidConfigFile':
            violator = exc.args[1]
            print(violator, file=sys.stderr)
            error = True
        elif msg == 'badKeyMapKey':
            # If we are here, then there was a non-empty keymap, and the parameter came from
            # the configuration file.
            violator = exc.args[1]
            print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
            error = True
        elif msg == 'badQuoteQual':
            # The bad quote qualifier came from the configuration file, not the addenda, since
            # we will have fixed any bad qualifiers in the addenda (which is populated by code).
            violator = exc.args[1]
            print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
            error = True
        elif msg == 'missingQuoteQual':
            violator = exc.args[1]
            print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
            error = True
        elif msg == 'paramNameTooLong':
            violator = exc.args[1]
            print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)
            error = True
        elif msg == 'unknownSection':
            violator = exc.args[1]
            print('Unknown section ' + "'" + violator + "' in configuration file.", file=sys.stderr)
            error = True
        else:
            # re-raise the exception
            raise

    if not error:
        if project_includes is not None:
            error = process_project_repos(project_includes)

    # Process addenda - these are parameters that are not configurable and must be set in the
    # NetDRMS build.
    if not error:
        iscfg = bool(0)
        for key, val in addenda.items():
            item = f'{key} {val}'
            print(f'addenda item {item}')
            if ignoreKeymap:
                keymapActual = None
            else:
                keymapActual = keymap
            ppRet = processParam(iscfg, item, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, 'defs')

    # Put information collected in platDict and machDict into mDefs. Must do this here, and not in processParam, since
    # we need to parse all platform-specific make variables before grouping them into platform categories.
    if not error:
        for plat in platDict:
            mDefsMake.extend(list('\nifeq ($(JSOC_MACHINE), linux_' + plat.lower() + ')'))
            for var in platDict[plat]:
                mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))
            mDefsMake.extend(list('\nendif\n'))

    if not error:
        for mach in machDict:
            mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))
            for var in machDict[mach]:
                mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))
            mDefsMake.extend(list('\nendif\n'))
    return error

def getMgrUIDLine(sums_manager, uidParam):
    error = False

    if sums_manager and len(sums_manager) > 0:
        cmd = [ 'id', '-u', shlex.quote(sums_manager) ]
        try:
            ret = check_output(cmd)
            uidParam['q:SUMS_MANAGER_UID'] = ret.decode("utf-8")
        except ValueError:
            print('Unable to run cmd: ' + cmd + '.')
            error = True
        except CalledProcessError:
            print('Command ' + "'" + cmd + "'" + ' ran improperly.')
            error = True

    return error

def isVersion(maj, min, majDef, minDef):
    res = 0

    if maj > majDef or (maj == majDef and min >= minDef):
        res = 1

    return res

def configureComps(defs, mDefs):
    rv = bool(0)
    autoConfig = bool(1)

    if 'AUTOSELCOMP' in defs:
        autoConfig = (not defs['AUTOSELCOMP'] == '0')

    if autoConfig:
        hasicc = bool(0)
        hasgcc = bool(0)
        hasifort = bool(0)
        hasgfort = bool(0)

        # Try icc.
        cmd = 'icc --version 2>&1'
        try:
            ret = check_output(cmd, shell=True)
            ret = ret.decode("utf-8")
        except CalledProcessError:
            print('Command ' + "'" + cmd + "'" + ' ran improperly.')
            rv = bool(1)

        if not rv:
            regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)[.](\d+)", re.DOTALL)
            matchobj = regexp.match(ret)
            if matchobj is None:
                raise Exception('unexpectedIccRet', ret)
            else:
                major = matchobj.group(1)
                minor = matchobj.group(2)
                if isVersion(int(major), int(minor), ICC_MAJOR, ICC_MINOR):
                    hasicc = bool(1)

        # Try gcc.
        if not hasicc:
            rv = bool(0)
            cmd = 'gcc -v 2>&1'
            try:
                ret = check_output(cmd, shell=True)
                ret = ret.decode("utf-8")
            except CalledProcessError:
                print('Command ' + "'" + cmd + "'" + ' ran improperly.')
                rv = bool(1)

            if not rv:
                regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
                matchobj = regexp.match(ret)
                if matchobj is None:
                    raise Exception('unexpectedGccRet', ret)
                else:
                    major = matchobj.group(1)
                    minor = matchobj.group(2)
                    if isVersion(int(major), int(minor), GCC_MAJOR, GCC_MINOR):
                        hasgcc = bool(1)

        # Try ifort.
        rv = bool(0)
        cmd = 'ifort --version 2>&1'
        try:
            ret = check_output(cmd, shell=True)
            ret = ret.decode("utf-8")
        except CalledProcessError:
            print('Command ' + "'" + cmd + "'" + ' ran improperly.')
            rv = bool(1)

        if not rv:
            regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)\.(\d+)", re.DOTALL)
            matchobj = regexp.match(ret)
            if matchobj is None:
                raise Exception('unexpectedIfortRet', ret)
            else:
                major = matchobj.group(1)
                minor = matchobj.group(2)
                if isVersion(int(major), int(minor), IFORT_MAJOR, IFORT_MINOR):
                    hasifort = bool(1)

        # Try gfortran
        if not hasifort:
            rv = bool(0)
            cmd = 'gfortran -v 2>&1'
            try:
                ret = check_output(cmd, shell=True)
                ret = ret.decode("utf-8")
            except CalledProcessError:
                print('Command ' + "'" + cmd + "'" + ' ran improperly.')
                rv = bool(1)

            if not rv:
                regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
                matchobj = regexp.match(ret)
                if matchobj is None:
                    raise Exception('unexpectedGfortranRet', ret)
                else:
                    major = matchobj.group(1)
                    minor = matchobj.group(2)
                    if isVersion(int(major), int(minor), GFORT_MAJOR, GFORT_MINOR):
                        hasgfort = bool(1)

        # Append the compiler make variables to the make file
        rv = bool(0)

        if not hasicc and not hasgcc:
            print('Fatal error: Acceptable C compiler not found! You will be unable to build the DRMS library.', file=sys.stderr)
            rv = bool(0) # Art - don't bail, we might be using drmsparams.py and not care about the rest of DRMS
        elif hasicc:
            mDefs.extend(list('\nCOMPILER = icc'))
            # mDefs.extend(list('\nICC_VERSION = blah'))
        else:
            mDefs.extend(list('\nCOMPILER = gcc'))

        if not hasifort and not hasgfort:
            print('Warning: Acceptable Fortran compiler not found! Fortran interface will not be built, and you will be unable to build Fortran modules.', file=sys.stderr)
        elif hasifort:
            mDefs.extend(list('\nFCOMPILER = ifort'))
        else:
            mDefs.extend(list('\nFCOMPILER = gfortran'))

        # Environment overrides. These get written, regardless of the disposition of auto-configuration.
        mDefs.extend(list('\nifneq ($(JSOC_COMPILER),)\n  COMPILER = $(JSOC_COMPILER)\nendif'))
        mDefs.extend(list('\nifneq ($(JSOC_FCOMPILER),)\n  FCOMPILER = $(JSOC_FCOMPILER)\nendif'))

    return rv

def writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
    rv = bool(0)

    # Merge mDefsGen, mDefsMake, and mDefsComps into a single string with compiler configuration first, general parameters next, then
    # make-specific make variables (e.g., third-party library information) last.
    mDefs = '\n# Compiler Selection\n' + ''.join(mDefsComps) + '\n\n# General Parameters\n' + ''.join(mDefsGen) + '\n\n# Parameters to Configure make\n' + ''.join(mDefsMake)

    try:
        with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout, open(pyfile, 'w') as pyout, open(shfile, 'w') as shout:
            # C file of macros
            print(C_PREFIX, file=cout)
            print('/* This file contains a set of preprocessor macros - one for each configuration parameter. */\n', file=cout)
            buf = '__' + base.upper() + '_H'
            print('#ifndef ' + buf, file=cout)
            print('#define ' + buf, file=cout)
            print(''.join(cDefs), file=cout)
            print('#endif', file=cout)

            # Make file of make variables
            print(PREFIX, file=mout)
            print('# This file contains a set of make-variable values. The first section contains compiler-selection variables, the second contains general configuration variables, and the third section contains variables that configure how make is run.', file=mout)
            print(mDefs, file=mout)

            # Perl module
            print(PERL_BINPATH, file=pout)
            print(PREFIX, file=pout)
            print('# This file contains a set of constants - one for each configuration parameter.\n', file=pout)
            print(PERL_INTIAL, file=pout)
            print(''.join(perlConstSection), file=pout)
            print(PERL_FXNS_A, file=pout)
            print('sub initialize', file=pout)
            print('{', file=pout)
            print('  my($self) = shift;', file=pout, end='')
            print('', file=pout)
            print(''.join(perlInitSection), file=pout)
            print('}\n', file=pout)
            print(PERL_FXNS_B, file=pout)

            # Python module
            print(PY_BINPATH, file=pyout)
            print(PREFIX, file=pyout)
            print('# This file contains a set of constants - one for each configuration parameter.\n', file=pyout)
            print(PY_ALL, file=pyout)
            print(''.join(pyConstSection), file=pyout)

            print(PY_ERROR_CLASSES, file=pyout)
            print(PY_FXNS_A, file=pyout, end='')
            print(''.join(pyInitSection), file=pyout)
            print(PY_FXNS_B, file=pyout)
            print(PY_FXNS_C, file=pyout)

            # Shell (bash) source file
            print(SH_BINPATH, file=shout)
            print(PREFIX, file=shout)
            print('# This file contains a set of variable assignments - one for each configuration parameter.\n', file=shout)
            print(''.join(shConstSection), file=shout)

    except IOError as exc:
        type, value, traceback = sys.exc_info()
        print(exc.strerror, file=sys.stderr)
        print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)
        rv = bool(1)

    return rv

def write_project_includes(project_includes_file, project_includes):
    error = False

    if project_includes_file is not None:
        try:
            with open(project_includes_file, 'w') as file_out:
                # Rules.mk
                print(PREFIX, file=file_out)
                print(RULESPREFIX, file=file_out)

                for dir_variable in project_includes:
                    print(dir_variable, file=file_out)
                    print(f'-include          $(SRCDIR)/$(dir)/Rules.mk', file=file_out)

                print('', file=file_out)
                print(RULESSUFFIX, file=file_out)

        except IOError as exc:
            print(f'unable to open {project_includes_file} for writing ({str(exc)})', file=sys.stderr)
            error = True

    return error

def generateSumRmCfg(defs):
    rv = bool(0)
    # ACK! Remember that Rick renamed these parameters. The ones in config.local are the aliases - do not use those.
    # Use the ones that those map to (defined in config.local.map).
    cFileTmp = defs['SUMLOG_BASEDIR'] + '/' + '.sum_rm.cfg.tmp'
    cFile = defs['SUMLOG_BASEDIR'] + '/' + 'sum_rm.cfg'

    # Write a temporary file sum_rm configuration file.
    try:
        with open(cFileTmp, 'w') as fout:
            # Print comment at the top of the configuration file.
            print(SUMRM_COMMENT, file=fout)
            print(SUMRM_DOC, file=fout)
            print(SUMRM_PARTN_PERCENT_FREE, file=fout)
            if 'SUMRM_PART_PERCENT_FREE' in defs:
                print('PART_PERCENT_FREE=' + defs['SUMRM_PART_PERCENT_FREE'], file=fout)
            else:
                print('PART_PERCENT_FREE=3', file=fout)

            print(SUMRM_SLEEP, file=fout)
            if 'SUMRM_SLEEP' in defs:
                print('SLEEP=' + defs['SUMRM_SLEEP'], file=fout)
            else:
                print('SLEEP=300', file=fout)

            print(SUMRM_LOG, file=fout)
            if 'SUMRM_LOG' in defs:
                print('LOG=' + defs['SUMRM_LOG'], file=fout)
            else:
                print('LOG=/tmp/sum_rm.log', file=fout)

            print(SUMRM_MAIL, file=fout)
            # No default for mail - don't send nothing to nobody unless the operator has asked for notifications.
            if 'SUMRM_MAIL' in defs:
                print('MAIL=' + defs['SUMRM_MAIL'], file=fout)
            else:
                print('# MAIL=president@whitehouse.gov', file=fout)

            print(SUMRM_NOOP, file=fout)
            if 'SUMRM_NOOP' in defs:
                print('NOOP=' + defs['SUMRM_NOOP'], file=fout)
            else:
                print('NOOP=0', file=fout)

            print(SUMRM_USER, file=fout)
            if 'SUMRM_USER' in defs:
                print('USER=' + defs['SUMRM_USER'], file=fout)
            else:
                print('USER=production', file=fout)

            print(SUMRM_NORUN, file=fout)
            # Default norun window is to have no such window. This can be accomplished by simply not providing either argument.
            if 'SUMRM_NORUN_START' in defs or 'SUMRM_NORUN_STOP' in defs:
                if 'SUMRM_NORUN_START' in defs:
                    print('NORUN_START=' + defs['SUMRM_NORUN_START'], file=fout)
                else:
                    print('NORUN_START=0', file=fout)
                if 'SUMRM_NORUN_STOP' in defs:
                    print('NORUN_STOP=' + defs['SUMRM_NORUN_STOP'], file=fout)
                else:
                    print('NORUN_STOP=0', file=fout)
            else:
                print('# NORUN_START=0', file=fout)
                print('# NORUN_STOP=0', file=fout)

    except OSError:
        print('Unable to open sum_rm temporary configuration file ' + cFileTmp + 'for writing.', file=sys.stderr)
        rv = bool(1)

    # If the content of the temporary file differs from the content of the existing configuration file, then overwrite
    # the original file. Otherwise, delete the temporary file
    if not rv:
        try:
            if filecmp.cmp(cFile, cFileTmp):
                # Files identical - delete temporary file
                try:
                    os.remove(cFileTmp)

                except OSError as exc:
                    print('Unable to remove temporary file ' + exc.filename + '.', file=sys.stderr)
                    print(exc.strerr, file=sys.stderr)
            else:
                # Replace original with temporary file
                try:
                    os.rename(cFileTmp, cFile)

                except OSError as exc:
                    print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                    print(exc.strerr, file=sys.stderr)
                    rv = bool(1)
        except OSError as exc:
            # One of the files doesn't exist.
            if exc.filename == cFile:
                # We are ok - there might be no configuration file yet.
                # Replace original with temporary file
                try:
                    os.rename(cFileTmp, cFile)

                except OSError as exc:
                    print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                    print(exc.strerr, file=sys.stderr)
                    rv = bool(1)
            else:
                # There is a problem with the temp file - bail.
                print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                print(exc.strerr, file=sys.stderr)
                rv = bool(1)

    return rv

def configureNet(cfgfile, cfile, mfile, pfile, pyfile, shfile, project_includes_file, base, keymap, createSumRmCfg):
    error = False

    defs = {}
    cDefs = list()
    mDefsGen = list()
    mDefsMake = list()
    mDefsComps = list()
    project_includes = []
    perlConstSection = list()
    perlInitSection = list()
    pyConstSection = list()
    pyInitSection = list()
    shConstSection = list()
    addenda = {}

    # There are three parameters that were not included in the original config.local parameter set, for some reason.
    # Due to this omission, then are not configurable, and must be set in the script.
    addenda['a:USER'] = 'NULL'
    addenda['a:PASSWD'] = 'NULL'
    addenda['p:DSDS_SUPPORT'] = '0'

    # This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
    addenda['a:BUILD_TYPE'] = 'NETDRMS' # Means a non-Stanford build. This will set two additional macros used by make:
                                        #   __LOCALIZED_DEFS__ and NETDRMS_BUILD. The former is to support legacy code
                                        #   which incorrectly used this macro, and the latter is for future use.
                                        #   __LOCALIZED_DEFS__ is deprecated and should not be used in new code.

    try:
        with open(cfgfile, 'r') as fin:
            # Process configuration parameters

            error = parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, project_includes, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
            if not error:
                # Must add a parameter for the SUMS_MANAGER UID (for some reason). This must be done after the
                # config file is processed since an input to getMgrUIDLine() is one of the config file's
                # parameter values.

                # CANNOT run this unless sunroom2 home directories are accessible
                uidParam = {}
                error = getMgrUIDLine(defs.get('SUMS_MANAGER', None), uidParam)
                if not error and len(uidParam) > 0:
                    error = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, None, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)

                # ignore the error where the SUMS manager UID cannot be determined
                error = False

            # Configure the compiler-selection make variables.
            if not error:
                error = configureComps(defs, mDefsComps)

            # Write out the parameter files.
            if not error:
                error = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)

            if not error:
                error = write_project_includes(project_includes_file, project_includes)

            # Write out the sum_rm.cfg file.
            if not error and createSumRmCfg:
                error = generateSumRmCfg(defs)
    except IOError as exc:
        print(exc.strerror, file=sys.stderr)
        print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
    except Exception as exc:
        if len(exc.args) >= 2:
            type, msg = exc.args
        else:
            # re-raise the exception
            raise

        if type == 'unexpectedIccRet':
            print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
            error = True
        elif type == 'unexpectedGccRet':
            print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
            error = True
        elif type == 'unexpectedIfortRet':
            print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
            error = True
        elif type == 'unexpectedGfortranRet':
            print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
            error = True
        else:
            # re-raise the exception
            raise

    return error

def configureSdp(cfgfile, cfile, mfile, pfile, pyfile, shfile, project_includes_file, base):
    error = False

    defs = {}
    cDefs = list()
    mDefsGen = list()
    mDefsMake = list()
    mDefsComps = list()
    project_includes = []
    perlConstSection = list()
    perlInitSection = list()
    pyConstSection = list()
    pyInitSection = list()
    shConstSection = list()

    addenda = {}

    # There are three parameters that were not included in the original config.local parameter set, for some reason.
    # Due to this omission, then are not configurable, and must be set in the script.
    addenda['a:USER'] = 'NULL'
    addenda['a:PASSWD'] = 'NULL'
    addenda['p:DSDS_SUPPORT'] = '1'

    # This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
    addenda['a:BUILD_TYPE'] = 'JSOC_SDP' # Means a Stanford build. This will set one additional macro used by make: JSOC_SDP_BUILD.

    try:
        with open(cfgfile, 'r') as fin:
            error = parseConfig(fin, None, addenda, defs, cDefs, mDefsGen, mDefsMake, project_includes, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)

            if not error:
                # Must add a parameter for the SUMS_MANAGER UID (for some reason)
                uidParam = {}
                error = getMgrUIDLine(defs.get('SUMS_MANAGER', None), uidParam)
                if not error and len(uidParam) > 0:
                    error = parseConfig(None, None, uidParam, defs, cDefs, mDefsGen, None, None, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)

                # ignore the error where the SUMS manager UID cannot be determined
                error = False

                # Configure the compiler-selection make variables.
                if not error:
                    error = configureComps(defs, mDefsComps)

                # Write out the parameter files.
                if not error:
                    error = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)

                if not error:
                    error = write_project_includes(project_includes_file, project_includes)

                # At Stanford, skip the creation of the sum_rm configuration file. config.local will still
                # have the SUMRM parameters, but they will not be used.
    except IOError as exc:
        print(exc.strerror, file=sys.stderr)
        print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
    except Exception as exc:
        if len(exc.args) >= 2:
            type = exc.args[0]
        else:
            # re-raise the exception
            raise

        if type == 'unexpectedIccRet':
            msg = exc.args[1]
            print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
            error = True
        elif type == 'unexpectedGccRet':
            msg = exc.args[1]
            print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
            error = True
        elif type == 'unexpectedIfortRet':
            msg = exc.args[1]
            print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
            error = True
        elif type == 'unexpectedGfortranRet':
            msg = exc.args[1]
            print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
            error = True
        else:
            # re-raise the exception
            raise

    return error

# Beginning of program
rv = RET_SUCCESS
net = bool(1)

# Parse arguments
if __name__ == "__main__":
    optD = GetArgs(sys.argv[1:])

if not(optD is None):
    # Ensure we are configuring a DRMS tree
    cdir = os.path.realpath(os.getcwd())
    versfile = cdir + '/base/' + VERS_FILE

    if not os.path.isfile(versfile):
        rv = RET_NOTDRMS

# Determine whether we are localizing a Stanford build, or a NetDRMS build. If configsdp.txt exists, then
# it is a Stanford build, otherwise it is a NetDRMS build.
if rv == RET_SUCCESS:
    stanfordFile = cdir + '/' + SDP_CFG
    if os.path.isfile(stanfordFile):
        net = bool(0)

    cfile = optD['dir'] + '/' + optD['base'] + '.h'
    mfile = optD['dir'] + '/' + optD['base'] + '.mk'
    pfile = optD['dir'] + '/' + optD['base'] + '.pm'
    pyfile = optD['dir'] + '/' + optD['base'] + '.py'
    shfile = optD['dir'] + '/' + optD['base'] + '.sh'
    project_includes_file = os.path.join(optD['dir'], 'includes.mk')

    if net:
        try:
            with open(NET_CFGMAP, 'r') as fin:
                regexpComm = re.compile(r"^\s*#")
                regexp = re.compile(r"^\s*(\S+)\s+(\w:\S+)")
                # Must map from config.local namespace to DRMS namespace (e.g., the names used for the C macros)
                keymap = {}
                for line in fin:
                    matchobj = regexpComm.match(line)
                    if not matchobj is None:
                        # Skip comment line
                        continue

                    matchobj = regexp.match(line)
                    if not(matchobj is None):
                        # We have a key-value line
                        key = matchobj.group(1)
                        val = matchobj.group(2)
                        keymap[key] = val
        except OSError:
            sys.stderr.write('Unable to read configuration map-file ' + NET_CFGMAP + '.')
            rv = bool(1)

        # We also need to set the UID of the SUMS manager. We have the name of the
        # SUMS manager (it is in the configuration file)
        configureNet(NET_CFG, cfile, mfile, pfile, pyfile, shfile, project_includes_file, optD['base'], keymap, 'server' in optD)
    else:
        # A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,
        # and then editing config.local. So, if config.local exists, use that.
        if os.path.isfile(cdir + '/' + NET_CFG):
            configureSdp(NET_CFG, cfile, mfile, pfile, pyfile, shfile, project_includes_file, optD['base'])
        else:
            configureSdp(SDP_CFG, cfile, mfile, pfile, pyfile, shfile, project_includes_file, optD['base'])
Revision:	1.33
Committed:	Wed Sep 14 15:59:13 2022 UTC (12 months, 1 week ago) by arta
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	HEAD
Changes since 1.32:	+139 -260 lines
Log Message:	Modifies make system to make it compatible with both the new git-repo structure and the old cvs structure (commit 1)