From ae5e6b3927fad1e3b6f8521ecf63ff35d35aedf8 Mon Sep 17 00:00:00 2001 From: Bryan Kowal Date: Tue, 12 Nov 2013 17:12:02 -0600 Subject: [PATCH] Issue #2540 - pure python module merge implementation Amend: remove dead LocalizationUtil.py file. Use os.path.join; catch Exception thrown by module merge Amend: remove module merge exception catching Change-Id: Idb9bc359d8b3dc1b6c4ccfd6247e02e2905bbea1 Former-commit-id: 48cc679bce4af23925a3a1ed938bdf29e699f478 [formerly 2029c34f24c78fbf20525f8571a8c52f545d6136] [formerly 48cc679bce4af23925a3a1ed938bdf29e699f478 [formerly 2029c34f24c78fbf20525f8571a8c52f545d6136] [formerly 7e35ea310facbdfef32a6259bc3c7c4828bf5502 [formerly c0595b49cab0d0ef636427ed3b31ea805f8adb43]]] Former-commit-id: 7e35ea310facbdfef32a6259bc3c7c4828bf5502 Former-commit-id: 4759e48c5c38b8f09f8a2a927bf3a6d9f109cc95 [formerly a2261b9ff35aa2b02ce135369b81610a1e9eb8f0] Former-commit-id: 79045d620b292e8a658a736f09d349781856aa9b --- .../{LocalizationUtil.py => ModuleUtil.py} | 53 +--- .../base/python/PythonOverrider.py | 135 +--------- .../base/python/PythonOverriderCore.py | 166 ++++++++++++ .../base/python/PythonOverriderPure.py | 254 ++++++++++++++++++ 4 files changed, 429 insertions(+), 179 deletions(-) rename edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/{LocalizationUtil.py => ModuleUtil.py} (52%) create mode 100644 edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverriderCore.py create mode 100644 edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverriderPure.py diff --git a/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/LocalizationUtil.py b/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/ModuleUtil.py similarity index 52% rename from edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/LocalizationUtil.py rename to edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/ModuleUtil.py index aaed389ccd..40bc2b9d39 100644 --- a/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/LocalizationUtil.py +++ b/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/ModuleUtil.py @@ -19,9 +19,8 @@ # # # -# Methods to aid in using the path manager from Python as well as other features. -# Contains internal classes to help with the transition from strings in Python -# to Java Localizationlevel and LocalizationType +# Provides a method to dynamically load a python module into memory. The method +# was relocated to this module from LocalizationUtil. # # # @@ -29,54 +28,10 @@ # # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- -# 03/12/13 mnash Initial Creation. -# 11/06/13 2086 bkowal Add the containing directory to the -# PYTHONPATH long enough to import -# the module. -# -# +# 11/11/13 bkowal Initial Creation. # - -import os -import imp -import sys - -from com.raytheon.uf.common.localization import LocalizationContext_LocalizationType as JavaLocalizationType, LocalizationContext_LocalizationLevel as JavaLocalizationLevel - -class LocalizationLevel(object): - ''' - @summary: Can use cmp() to compare the levels, and can use values() to get all possible levels - ''' - @staticmethod - def cmp(level1, level2): - return JavaLocalizationLevel.compare(level1,level2) +import os, sys, imp - @staticmethod - def values(): - jvals = JavaLocalizationLevel.values() - vals = list() - for val in jvals : - vals.append(val.name()) - return vals - - @staticmethod - def valueOf(value): - return JavaLocalizationLevel.valueOf(value) - -class LocalizationType(object): - - @staticmethod - def values(): - jvals = JavaLocalizationType.values() - vals = list() - for val in jvals : - vals.append(val.name()) - return vals - - @staticmethod - def valueOf(value): - return JavaLocalizationType.valueOf(value) - def loadModule(filename): ''' @param filename: the fully qualified name of the file diff --git a/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverrider.py b/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverrider.py index 003d5e5c00..2d34920022 100644 --- a/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverrider.py +++ b/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverrider.py @@ -30,15 +30,13 @@ # 03/12/13 mnash Initial Creation. # 11/04/13 2086 bkowal Updated to merge classes - both legacy and non-legacy. # Minimum to Maximum level of retrieval can now be specified. +# 11/12/13 2540 bkowal Relocated common methods to PythonOverriderCore.py. # # # -import os -import imp -import types +import PythonOverriderCore from PathManager import PathManager -import LocalizationUtil def importModule(name, loctype='COMMON_STATIC', level=None): """ @@ -56,134 +54,11 @@ def importModule(name, loctype='COMMON_STATIC', level=None): pathManager = PathManager() tieredFiles = pathManager.getTieredLocalizationFile(loctype, name) availableLevels = pathManager.getAvailableLevels() - if level == None: - levels = availableLevels - else: - # ensure that the specified level is actually a legitimate level - if level not in availableLevels: - raise LookupError('An invalid level has been specified!') - - levels = [] - try: - levelIdx = availableLevels.index(level) - levels = availableLevels[:levelIdx + 1] - except ValueError: - # ignore; the exception should never be thrown, we verify that the specified level - # is valid in the previous if statement - pass + levels = PythonOverriderCore._buildLocalizationLevelsList(availableLevels, level) lfiles = [] for _level in levels : if _level in tieredFiles: lfiles.append(tieredFiles[_level].getPath()) - themodule = _internalOverride(lfiles) - return themodule - -def _internalOverride(files): - """ - Takes the files and overrides them - - Args: - files : the files that are to be overridden - - Returns: - a new module that contains all the necessary elements - """ - themodule = imp.new_module('tmpmodule') - # modules = list of all the modules - for module in files : - # load each module, temporarily - tmpmodule = LocalizationUtil.loadModule(module) - themodule = _combineMembers(tmpmodule, themodule) - return themodule - -def _combineMembers(tocombine, combinationresult): - for attr in dir(tocombine): - if attr.startswith('__') or attr.startswith('_') or isType(attr, types.BuiltinFunctionType): - # skip - continue - - # is the element a class? - if isType(getattr(tocombine, attr), types.ClassType): - combinationresult = _mergeClasses(tocombine, combinationresult, attr) - else: - # absolute override - combinationresult = _mergeAttributes(tocombine, combinationresult, attr) - - return combinationresult - -def _mergeClasses(source, target, className): - sourceClass = getattr(source, className) - targetClass = getattr(target, className, None) - - if (targetClass == None): - return _mergeAttributes(source, target, className) - - legacyMode = (hasattr(sourceClass, '__class__') == False) - - # verify that both classes are either legacy for current style. - if ((hasattr(targetClass, '__class__') == False) != legacyMode): - raise Exception("A legacy python class cannot be merged with a non-legacy python class!") - - # ensure that the classes are not exactly the same (breaks the legacy merge). - if compareClasses(sourceClass, targetClass): - # nothing to merge - return target - - for attr in dir(sourceClass): - # include private attributes because this is a class? - # methods cannot just be merged into a class, so skip them. - if isType(attr, types.BuiltinFunctionType) or isType(attr, types.MethodType) or \ - attr.startswith('__') or attr.startswith('_'): - continue - - # do we need to worry about nested classes? - if isType(getattr(sourceClass, attr), types.ClassType): - target = _mergeClasses(source, target, attr) - - attributeName = className + '.' + attr - target = _mergeAttributes(source, target, attributeName) - - # complete the merge / override of methods. - exec(_buildMergeDirective(className, legacyMode)) - return _mergeAttributes(source, target, className) - -def _buildMergeDirective(className, legacyMode): - if (legacyMode): - return 'source.' + className + '.__bases__ = (target.' + className + ',)' - else: - return 'source.' + className + ' = type("' + className + \ - '", (target.' + className + ',), dict(source.' + className + '.__dict__))' - -def isType(object, type): - return type(object) == type - -def compareClasses(clazz1, clazz2): - clazz1Attr = dir(clazz1) - clazz2Attr = dir(clazz2) - - if (len(clazz1Attr) != len(clazz2Attr)): - return False - - i = 0 - while i < len(clazz1Attr): - # compare the names - if (clazz1Attr[i] != clazz2Attr[i]): - return False - - # compare the attributes directly - attr1 = getattr(clazz1, clazz1Attr[i]) - attr2 = getattr(clazz2, clazz2Attr[i]) - if (attr1 != attr2): - return False - - i += 1 - - return True - -def _mergeAttributes(source, target, attributeName): - - mergeDirective = 'target.' + attributeName + ' = source.' + attributeName - exec(mergeDirective) - - return target \ No newline at end of file + themodule = PythonOverriderCore._internalOverride(lfiles) + return themodule \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverriderCore.py b/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverriderCore.py new file mode 100644 index 0000000000..82c08542ee --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverriderCore.py @@ -0,0 +1,166 @@ +# # +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +# # + +# +# Used to merge two ore more python modules and returns a new module. This set of methods +# are compatible with and used by both the Jep Python Overrider solution and the Pure +# (outside of Jep, no Java dependencies) Python Overrider solution. +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 11/12/13 bkowal Initial Creation. +# +# +# + +import os, imp, types +import ModuleUtil + +def _internalOverride(files): + """ + Takes the files and overrides them + + Args: + files : the files that are to be overridden + + Returns: + a new module that contains all the necessary elements + """ + themodule = imp.new_module('tmpmodule') + # modules = list of all the modules + for module in files : + # load each module, temporarily + tmpmodule = ModuleUtil.loadModule(module) + themodule = _combineMembers(tmpmodule, themodule) + return themodule + +def _buildLocalizationLevelsList(availableLevels, desiredLevel): + if desiredLevel == None: + return availableLevels + else: + # ensure that the specified level is actually a legitimate level + if desiredLevel not in availableLevels: + raise LookupError('An invalid level has been specified!') + + levels = [] + try: + levelIdx = availableLevels.index(desiredLevel) + levels = availableLevels[:levelIdx + 1] + except ValueError: + # ignore; the exception should never be thrown, we verify that the specified level + # is valid in the previous if statement + pass + + return levels + +def _combineMembers(tocombine, combinationresult): + for attr in dir(tocombine): + if attr.startswith('__') or attr.startswith('_') or isinstance(attr, types.BuiltinFunctionType): + # skip + continue + + # is the element a class? + if isinstance(getattr(tocombine, attr), types.ClassType) \ + or isinstance(getattr(tocombine, attr), types.TypeType): + combinationresult = _mergeClasses(tocombine, combinationresult, attr) + else: + # absolute override + combinationresult = _mergeAttributes(tocombine, combinationresult, attr) + + return combinationresult + +def _mergeClasses(source, target, className): + sourceClass = getattr(source, className) + targetClass = getattr(target, className, None) + + if (targetClass == None): + return _mergeAttributes(source, target, className) + + legacyMode = (hasattr(sourceClass, '__class__') == False) + + # verify that both classes are either legacy for current style. + if ((hasattr(targetClass, '__class__') == False) != legacyMode): + raise Exception("A legacy python class cannot be merged with a non-legacy python class!") + + # ensure that the classes are not exactly the same (breaks the legacy merge). + if _compareClasses(sourceClass, targetClass): + # nothing to merge + return target + + for attr in dir(sourceClass): + # include private attributes because this is a class? + # methods cannot just be merged into a class, so skip them. + if isinstance(attr, types.BuiltinFunctionType) \ + or isinstance(attr, types.MethodType) \ + or attr.startswith('__') or attr.startswith('_'): + continue + + # do we need to worry about nested classes? + if isinstance(getattr(sourceClass, attr), types.ClassType) \ + or isinstance(getattr(sourceClass, attr), types.TypeType): + target = _mergeClasses(source, target, attr) + + attributeName = className + '.' + attr + target = _mergeAttributes(source, target, attributeName) + + # complete the merge / override of methods. + exec(_buildMergeDirective(className, legacyMode)) + return _mergeAttributes(source, target, className) + +def _buildMergeDirective(className, legacyMode): + if (legacyMode): + return 'source.' + className + '.__bases__ = (target.' + className + ',)' + else: + return 'source.' + className + ' = type("' + className + \ + '", (target.' + className + ',), dict(source.' + className + '.__dict__))' + +def _compareClasses(clazz1, clazz2): + clazz1Attr = dir(clazz1) + clazz2Attr = dir(clazz2) + + if (len(clazz1Attr) != len(clazz2Attr)): + return False + + i = 0 + while i < len(clazz1Attr): + # compare the names + if (clazz1Attr[i] != clazz2Attr[i]): + return False + + # compare the attributes directly + attr1 = getattr(clazz1, clazz1Attr[i]) + attr2 = getattr(clazz2, clazz2Attr[i]) + if (attr1 != attr2): + return False + + i += 1 + + return True + +def _mergeAttributes(source, target, attributeName): + + mergeDirective = 'target.' + attributeName + ' = source.' + attributeName + exec(mergeDirective) + + return target \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverriderPure.py b/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverriderPure.py new file mode 100644 index 0000000000..cf5e72386b --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.localization.python/utility/common_static/base/python/PythonOverriderPure.py @@ -0,0 +1,254 @@ +# # +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +# # + +# +# The pure python module override solution. Merges multiple python modules retrieved +# from localization. +# +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 03/12/13 bkowal Initial Creation. +# +# +# + +import os, tempfile, shutil +import numpy +import PythonOverriderCore +from ufpy import ThriftClient +from dynamicserialize.dstypes.com.raytheon.uf.common.auth.resp import SuccessfulExecution +from dynamicserialize.dstypes.com.raytheon.uf.common.localization import LocalizationContext +from dynamicserialize.dstypes.com.raytheon.uf.common.localization.msgs import ListUtilityCommand +from dynamicserialize.dstypes.com.raytheon.uf.common.localization.msgs import UtilityRequestMessage +from dynamicserialize.dstypes.com.raytheon.uf.common.localization import LocalizationLevel +from dynamicserialize.dstypes.com.raytheon.uf.common.localization import LocalizationType +from dynamicserialize.dstypes.com.raytheon.uf.common.localization.stream import LocalizationStreamGetRequest + +BUFFER_SIZE = 512 * 1024 +availableLevels = ['BASE', 'CONFIGURED', 'SITE', 'USER'] + +def importModule(name, localizationHost, localizationPort, localizedSite, localizationUser=None, + loctype='COMMON_STATIC', level=None): + ''' + @param name: the name of the localization file + @param localizationHost: the EDEX server that the localization should be + retrieved from + @param localizationPort: the port that will be used to connect to the + EDEX server + @param localizedSite: the site that localization information should be + retrieved for (if applicable) + @param localizationUser: the user that localization information should + be retrieved for (if applicable) + @param loctype: the type of localization files to retrieve + @param level: the minimum level that localization files should be retrieved + for + @return: the merged module + @summary: this is the pure python (no jep dependencies) version of the python overrider + ''' + + # determine which localization levels files need to be retrieved for + levels = PythonOverriderCore._buildLocalizationLevelsList(availableLevels, level) + + # create a thrift instance + thrift = ThriftClient.ThriftClient(localizationHost, localizationPort, '/services') + + # retrieve a list of the localization files that will need to be merged + serverResponse = \ + _executeLocalizationFileListRetrieval(name, levels, localizedSite, + localizationUser, loctype, thrift) + # download the localization files + lfiles = _downloadLocalizationFiles(serverResponse, thrift, name) + # complete the module merge + return _executeModuleMerge(lfiles) + +def _executeLocalizationFileListRetrieval(filename, localizationLevels, localizedSite, + localizationUser, loctype, thrift): + ''' + @param filename: the name of the localization file + @param localizationLevels: a list of localization levels that should be checked + for the specified localization file + @param localizedSite: the site that localization information should be + retrieved for (if applicable) + @param localizationUser: the user that localization information should + be retrieved for (if applicable) + @param loctype: the type of localization files to retrieve + @param thrift: an instance of the thrift client used to communicate + with EDEX + @return: a list of the localization files associated with the specified + file name wrapped in a server response + @summary: this function will execute a list utility command via thrift to + retrieve a list of localization files that match the specified file name, + that match the specified localization type, and that can be found within the + specified localization levels. + ''' + + directory = os.path.dirname(filename) + + cmds = [] + + # prepare the localization type + localizationType = LocalizationType(loctype) + + # build the request message + req = UtilityRequestMessage() + for level in localizationLevels: + cmd = ListUtilityCommand() + cmd.setSubDirectory(directory) + cmd.setRecursive(False) + cmd.setFilesOnly(True) + cmd.setLocalizedSite(localizedSite) + + # prepare the localization level + localizationLevel = LocalizationLevel(level) + + # create a localization context + localizationContext = LocalizationContext() + localizationContext.setLocalizationType(localizationType) + localizationContext.setLocalizationLevel(localizationLevel) + if level in ['CONFIGURED', 'SITE' ]: + localizationContext.setContextName(localizedSite) + elif level == 'USER': + localizationContext.setContextName(localizationUser) + + # build the utility command + cmd.setContext(localizationContext) + cmds.append(cmd) + + # add the command(s) to the request + req.setCommands(cmds) + + try: + serverResponse = thrift.sendRequest(req) + except Exception, e: + raise RuntimeError, 'Could not retrieve localization file list: ' + str(e) + + return serverResponse + +def _downloadLocalizationFiles(serverResponse, thrift, name): + ''' + @param serverResponse: a list of localization files that will need to + be downloaded as well the localization context associated with + each file + @param thrift: an instance of the thrift client used to communicate + with EDEX + @param name: the name of the localization file to download. used + to verify that .pyc files are not retrieved + @return: a list of the localization files that have been downloaded + @summary: this function will loop through the server response, + execute a function that will retrieve the localization file + from the server as bytes, and write the bytes to a temporary + file in a dynamically generated location. + ''' + + lfiles = [] + + for response in serverResponse.getResponses(): + for entry in response.getEntries(): + filename = entry.getFileName() + if filename == name: + # create a local copy of the file + localizationContext = entry.getContext() + bytes = _retrieveFileFromServer(localizationContext, filename, thrift) + # create the temporary directory + directoryName = tempfile.mkdtemp() + filename = os.path.split(name)[1] + fileToWrite = os.path.join(directoryName, filename) + try: + with open(fileToWrite, 'wb') as fileHandle: + fileHandle.write(bytes) + except: + # remove any files that have successfully + # been created. + _removeTemporaryLocalizationFiles(lfiles) + # fail + raise IOError('Failed to create a local copy of the localization file: ' + filename) + + lfiles.append(fileToWrite) + break + + return lfiles + +def _executeModuleMerge(fileList): + ''' + @param fileList: a list of python modules that will be merged + @return: the merged module + @summary: this function will run the module merge + ''' + + # perform the module merge + try: + themodule = PythonOverriderCore._internalOverride(fileList) + finally: + _removeTemporaryLocalizationFiles(fileList) + + return themodule + +def _removeTemporaryLocalizationFiles(files): + ''' + @param files: a list files to remove + @summary: this function will remove the temporary localization + files and directories + ''' + + for file in files: + directory = os.path.dirname(file) + shutil.rmtree(directory, True) + +def _retrieveFileFromServer(localizationContext, filename, thrift): + ''' + @param localizationContext: the localization context associated + with the file that will be downloaded + @param filename: the name of the file to download + @param thrift: an instance of the thrift client used to communicate + with EDEX + @return: the bytes associated with the file that was downloaded + @summary: + ''' + + request = LocalizationStreamGetRequest() + request.setOffset(0) + request.setNumBytes(BUFFER_SIZE) + request.setContext(localizationContext) + request.setMyContextName(localizationContext.getContextName()) + request.setFileName(filename) + + bytes = numpy.array([], numpy.int8) + finished = False + while (not finished): + serverResponse = thrift.sendRequest(request) + if not isinstance(serverResponse, SuccessfulExecution): + message = "" + if hasattr(serverResponse, 'getMessage'): + message = serverResponse.getMessage() + raise RuntimeError(message) + serverResponse = serverResponse.getResponse() + # serverResponse will be returned as a LocalizationStreamPutRequest + # object. we'll use its methods to read back the serialized file + # data. + # bytes get returned to us as an numpy.ndarray + bytes = numpy.append(bytes, serverResponse.getBytes()) + request.setOffset(request.getOffset() + len(bytes)) + finished = serverResponse.getEnd() + + return bytes \ No newline at end of file