ddb3fd58dc
-Had to update the awips-ade and awips-devel dockers -Update rpm building to include awips2-python-awips (which includes awips, dynamicserialize, thrift packages), and remove rpm install dependencies on ufpy, dynamicserialize, and thrift -Update LDM scripts and edexBridge to use awips instead of ufpy -Update multiple python scripts across several repos to use awips instead of ufpy -awips2 -awips2-core -awips2-nativelib -awips2-ncep -awips2-rpm
260 lines
No EOL
10 KiB
Python
260 lines
No EOL
10 KiB
Python
# #
|
|
# This software was developed and / or modified by Raytheon Company,
|
|
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
|
#
|
|
# U.S. EXPORT CONTROLLED TECHNICAL DATA
|
|
# This software product contains export-restricted data whose
|
|
# export/transfer/disclosure is restricted by U.S. law. Dissemination
|
|
# to non-U.S. persons whether in the United States or abroad requires
|
|
# an export license or other authorization.
|
|
#
|
|
# Contractor Name: Raytheon Company
|
|
# Contractor Address: 6825 Pine Street, Suite 340
|
|
# Mail Stop B8
|
|
# Omaha, NE 68106
|
|
# 402.291.0100
|
|
#
|
|
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
|
# further licensing information.
|
|
# #
|
|
|
|
#
|
|
# The pure python module override solution. Merges multiple python modules retrieved
|
|
# from localization.
|
|
#
|
|
#
|
|
#
|
|
# SOFTWARE HISTORY
|
|
#
|
|
# Date Ticket# Engineer Description
|
|
# ------------ ---------- ----------- --------------------------
|
|
# 03/12/13 bkowal Initial Creation.
|
|
# 02/17/14 2712 bkowal Provide a default value for localization site.
|
|
# 03/19/14 2929 bkowal 'REGION' is now recognized as a valid localization level.
|
|
# 02/04/19 mjames@ucar Removed REGION.
|
|
#
|
|
|
|
#
|
|
# TODO: Update to use the HTTP Rest Service to retrieve localization files instead
|
|
# of thrift.
|
|
#
|
|
|
|
import os, tempfile, shutil
|
|
import numpy
|
|
import PythonOverriderCore
|
|
from awips import ThriftClient
|
|
from dynamicserialize.dstypes.com.raytheon.uf.common.auth.resp import SuccessfulExecution
|
|
from dynamicserialize.dstypes.com.raytheon.uf.common.localization import LocalizationContext
|
|
from dynamicserialize.dstypes.com.raytheon.uf.common.localization.msgs import ListUtilityCommand
|
|
from dynamicserialize.dstypes.com.raytheon.uf.common.localization.msgs import UtilityRequestMessage
|
|
from dynamicserialize.dstypes.com.raytheon.uf.common.localization import LocalizationLevel
|
|
from dynamicserialize.dstypes.com.raytheon.uf.common.localization import LocalizationType
|
|
from dynamicserialize.dstypes.com.raytheon.uf.common.localization.stream import LocalizationStreamGetRequest
|
|
|
|
BUFFER_SIZE = 512 * 1024
|
|
availableLevels = ['BASE', 'CONFIGURED', 'SITE', 'USER']
|
|
|
|
def importModule(name, localizationHost, localizationPort, localizedSite=None, localizationUser=None,
|
|
loctype='COMMON_STATIC', level=None):
|
|
'''
|
|
@param name: the name of the localization file
|
|
@param localizationHost: the EDEX server that the localization should be
|
|
retrieved from
|
|
@param localizationPort: the port that will be used to connect to the
|
|
EDEX server
|
|
@param localizedSite: the site that localization information should be
|
|
retrieved for (if applicable)
|
|
@param localizationUser: the user that localization information should
|
|
be retrieved for (if applicable)
|
|
@param loctype: the type of localization files to retrieve
|
|
@param level: the minimum level that localization files should be retrieved
|
|
for
|
|
@return: the merged module
|
|
@summary: this is the pure python (no jep dependencies) version of the python overrider
|
|
'''
|
|
|
|
# determine which localization levels files need to be retrieved for
|
|
levels = PythonOverriderCore._buildLocalizationLevelsList(availableLevels, level)
|
|
|
|
# create a thrift instance
|
|
thrift = ThriftClient.ThriftClient(localizationHost, localizationPort, '/services')
|
|
|
|
# retrieve a list of the localization files that will need to be merged
|
|
serverResponse = \
|
|
_executeLocalizationFileListRetrieval(name, levels, localizedSite,
|
|
localizationUser, loctype, thrift)
|
|
# download the localization files
|
|
lfiles = _downloadLocalizationFiles(serverResponse, thrift, name)
|
|
# complete the module merge
|
|
return _executeModuleMerge(lfiles)
|
|
|
|
def _executeLocalizationFileListRetrieval(filename, localizationLevels, localizedSite,
|
|
localizationUser, loctype, thrift):
|
|
'''
|
|
@param filename: the name of the localization file
|
|
@param localizationLevels: a list of localization levels that should be checked
|
|
for the specified localization file
|
|
@param localizedSite: the site that localization information should be
|
|
retrieved for (if applicable)
|
|
@param localizationUser: the user that localization information should
|
|
be retrieved for (if applicable)
|
|
@param loctype: the type of localization files to retrieve
|
|
@param thrift: an instance of the thrift client used to communicate
|
|
with EDEX
|
|
@return: a list of the localization files associated with the specified
|
|
file name wrapped in a server response
|
|
@summary: this function will execute a list utility command via thrift to
|
|
retrieve a list of localization files that match the specified file name,
|
|
that match the specified localization type, and that can be found within the
|
|
specified localization levels.
|
|
'''
|
|
|
|
directory = os.path.dirname(filename)
|
|
|
|
cmds = []
|
|
|
|
# prepare the localization type
|
|
localizationType = LocalizationType(loctype)
|
|
|
|
# build the request message
|
|
req = UtilityRequestMessage()
|
|
for level in localizationLevels:
|
|
cmd = ListUtilityCommand()
|
|
cmd.setSubDirectory(directory)
|
|
cmd.setRecursive(False)
|
|
cmd.setFilesOnly(True)
|
|
cmd.setLocalizedSite(localizedSite)
|
|
|
|
# prepare the localization level
|
|
localizationLevel = LocalizationLevel(level)
|
|
|
|
# create a localization context
|
|
localizationContext = LocalizationContext()
|
|
localizationContext.setLocalizationType(localizationType)
|
|
localizationContext.setLocalizationLevel(localizationLevel)
|
|
if level in ['CONFIGURED', 'SITE' ]:
|
|
localizationContext.setContextName(localizedSite)
|
|
elif level == 'USER':
|
|
localizationContext.setContextName(localizationUser)
|
|
|
|
# build the utility command
|
|
cmd.setContext(localizationContext)
|
|
cmds.append(cmd)
|
|
|
|
# add the command(s) to the request
|
|
req.setCommands(cmds)
|
|
|
|
try:
|
|
serverResponse = thrift.sendRequest(req)
|
|
except Exception as e:
|
|
raise RuntimeError('Could not retrieve localization file list: ' + str(e))
|
|
|
|
return serverResponse
|
|
|
|
def _downloadLocalizationFiles(serverResponse, thrift, name):
|
|
'''
|
|
@param serverResponse: a list of localization files that will need to
|
|
be downloaded as well the localization context associated with
|
|
each file
|
|
@param thrift: an instance of the thrift client used to communicate
|
|
with EDEX
|
|
@param name: the name of the localization file to download. used
|
|
to verify that .pyc files are not retrieved
|
|
@return: a list of the localization files that have been downloaded
|
|
@summary: this function will loop through the server response,
|
|
execute a function that will retrieve the localization file
|
|
from the server as bytes, and write the bytes to a temporary
|
|
file in a dynamically generated location.
|
|
'''
|
|
|
|
lfiles = []
|
|
|
|
for response in serverResponse.getResponses():
|
|
for entry in response.getEntries():
|
|
filename = entry.getFileName()
|
|
if filename == name:
|
|
# create a local copy of the file
|
|
localizationContext = entry.getContext()
|
|
bytes = _retrieveFileFromServer(localizationContext, filename, thrift)
|
|
# create the temporary directory
|
|
directoryName = tempfile.mkdtemp()
|
|
filename = os.path.split(name)[1]
|
|
fileToWrite = os.path.join(directoryName, filename)
|
|
try:
|
|
with open(fileToWrite, 'wb') as fileHandle:
|
|
fileHandle.write(bytes)
|
|
except:
|
|
# remove any files that have successfully
|
|
# been created.
|
|
_removeTemporaryLocalizationFiles(lfiles)
|
|
# fail
|
|
raise IOError('Failed to create a local copy of the localization file: ' + filename)
|
|
|
|
lfiles.append(fileToWrite)
|
|
break
|
|
|
|
return lfiles
|
|
|
|
def _executeModuleMerge(fileList):
|
|
'''
|
|
@param fileList: a list of python modules that will be merged
|
|
@return: the merged module
|
|
@summary: this function will run the module merge
|
|
'''
|
|
|
|
# perform the module merge
|
|
try:
|
|
themodule = PythonOverriderCore._internalOverride(fileList)
|
|
finally:
|
|
_removeTemporaryLocalizationFiles(fileList)
|
|
|
|
return themodule
|
|
|
|
def _removeTemporaryLocalizationFiles(files):
|
|
'''
|
|
@param files: a list files to remove
|
|
@summary: this function will remove the temporary localization
|
|
files and directories
|
|
'''
|
|
|
|
for file in files:
|
|
directory = os.path.dirname(file)
|
|
shutil.rmtree(directory, True)
|
|
|
|
def _retrieveFileFromServer(localizationContext, filename, thrift):
|
|
'''
|
|
@param localizationContext: the localization context associated
|
|
with the file that will be downloaded
|
|
@param filename: the name of the file to download
|
|
@param thrift: an instance of the thrift client used to communicate
|
|
with EDEX
|
|
@return: the bytes associated with the file that was downloaded
|
|
@summary:
|
|
'''
|
|
|
|
request = LocalizationStreamGetRequest()
|
|
request.setOffset(0)
|
|
request.setNumBytes(BUFFER_SIZE)
|
|
request.setContext(localizationContext)
|
|
request.setMyContextName(localizationContext.getContextName())
|
|
request.setFileName(filename)
|
|
|
|
bytes = numpy.array([], numpy.int8)
|
|
finished = False
|
|
while (not finished):
|
|
serverResponse = thrift.sendRequest(request)
|
|
if not isinstance(serverResponse, SuccessfulExecution):
|
|
message = ""
|
|
if hasattr(serverResponse, 'getMessage'):
|
|
message = serverResponse.getMessage()
|
|
raise RuntimeError(message)
|
|
serverResponse = serverResponse.getResponse()
|
|
# serverResponse will be returned as a LocalizationStreamPutRequest
|
|
# object. we'll use its methods to read back the serialized file
|
|
# data.
|
|
# bytes get returned to us as an numpy.ndarray
|
|
bytes = numpy.append(bytes, serverResponse.getBytes())
|
|
request.setOffset(request.getOffset() + len(bytes))
|
|
finished = serverResponse.getEnd()
|
|
|
|
return bytes |