mirror of
https://github.com/Unidata/python-awips.git
synced 2025-02-24 06:57:56 -05:00
456 lines
18 KiB
Python
456 lines
18 KiB
Python
##
|
|
##
|
|
|
|
#
|
|
# Library for accessing localization files from python.
|
|
#
|
|
# SOFTWARE HISTORY
|
|
#
|
|
# Date Ticket# Engineer Description
|
|
# --------- -------- --------- --------------------------
|
|
# 08/09/17 5731 bsteffen Initial Creation.
|
|
|
|
|
|
import urllib2
|
|
from json import load as loadjson
|
|
from xml.etree.ElementTree import parse as parseXml
|
|
from base64 import b64encode
|
|
try:
|
|
from StringIO import StringIO
|
|
except ImportError:
|
|
from io import StringIO
|
|
from getpass import getuser
|
|
import dateutil.parser
|
|
import contextlib
|
|
import os
|
|
from urlparse import urlunparse, urljoin
|
|
|
|
NON_EXISTENT_CHECKSUM = 'NON_EXISTENT_CHECKSUM'
|
|
DIRECTORY_CHECKSUM = 'DIRECTORY_CHECKSUM'
|
|
|
|
class LocalizationFileVersionConflictException(Exception):
|
|
pass
|
|
|
|
class LocalizationFileDoesNotExistException(Exception):
|
|
pass
|
|
|
|
class LocalizationFileIsNotDirectoryException(Exception):
|
|
pass
|
|
|
|
class LocalizationContext(object):
|
|
"""A localization context defines the scope of a localization file.
|
|
|
|
For example the base localization context includes all the default files
|
|
installed with EDEX, while a particular user context has custom files for
|
|
that user.
|
|
|
|
A localization context consists of a level and name. The level defines what
|
|
kind of entity this context is valid for, such as 'base', 'site', or 'user'.
|
|
The name identifies the specific entity, for example the name of a 'user'
|
|
level context is usually the username. The 'base' level does not have a name
|
|
because there cannot be only one 'base' context.
|
|
|
|
Attributes:
|
|
level: the localization level
|
|
name: the context name
|
|
"""
|
|
def __init__(self, level="base", name=None, type="common_static"):
|
|
if level != "base":
|
|
assert name is not None
|
|
self.level = level
|
|
self.name = name
|
|
self.type = type
|
|
def isBase(self):
|
|
return self.level == "base"
|
|
def _getUrlComponent(self):
|
|
if self.isBase():
|
|
return self.type + '/' + "base/"
|
|
else:
|
|
return self.type + '/' + self.level + '/' + self.name + '/'
|
|
def __str__(self):
|
|
if self.isBase():
|
|
return self.type + ".base"
|
|
else:
|
|
return self.type + "." + self.level + "." + self.name
|
|
def __eq__(self, other):
|
|
return self.level == other.level and \
|
|
self.name == other.name and \
|
|
self.type == other.type
|
|
def __hash__(self):
|
|
return hash((self.level, self.name, self.type))
|
|
|
|
class _LocalizationOutput(StringIO):
|
|
"""A file-like object for writing a localization file.
|
|
|
|
The contents being written are stored in memory and written to a
|
|
localization server only when the writing is finished.
|
|
|
|
This object should be used as a context manager, a save operation will be
|
|
executed if the context exits with no errors. If errors occur the partial
|
|
contents are abandoned and the server is unchanged.
|
|
|
|
It is also possible to save the contents to the server with the save()
|
|
method.
|
|
"""
|
|
def __init__(self, manager, file):
|
|
StringIO.__init__(self)
|
|
self._manager = manager
|
|
self._file = file
|
|
def save(self):
|
|
"""Send the currently written contents to the server."""
|
|
request = self._manager._buildRequest(self._file.context, self._file.path, method="PUT")
|
|
|
|
request.add_data(self.getvalue())
|
|
request.add_header("If-Match", self._file.checksum)
|
|
try:
|
|
urllib2.urlopen(request)
|
|
except urllib2.HTTPError as e:
|
|
if e.code == 409:
|
|
raise LocalizationFileVersionConflictException(e.read())
|
|
else:
|
|
raise e
|
|
def __enter__(self):
|
|
return self
|
|
def __exit__(self, exc_type, exc_value, traceback):
|
|
if exc_type is None:
|
|
self.save()
|
|
def __str__(self):
|
|
return '<' + self.__class__.__name__ + " for " + str(self._file) + '>'
|
|
|
|
class LocalizationFile(object):
|
|
"""A specific file stored in localization.
|
|
|
|
A localization file is uniquely defined by the context and path. There can
|
|
only be one valid file for that path and localization at a time. To access
|
|
the contents of the file use the open method.
|
|
|
|
Attributes:
|
|
context: A LocalizationContext
|
|
path: A path to this file
|
|
checksum: A string representation of a checksum generated from the file contents.
|
|
timnestamp: A datetime.datetime object indicating when the file was last modified.
|
|
"""
|
|
def __init__(self, manager, context, path, checksum, timestamp):
|
|
"""Initialize a LocalizationFile with the given manager and attributes.
|
|
|
|
Args:
|
|
manager: A LocalizationFileManager to assist with server communication
|
|
context: A LocalizationContext
|
|
path: A path to this file
|
|
checksum: A string representation of a checksum generated from the file contents.
|
|
timnestamp: A datetime.datetime object indicating when the file was last modified.
|
|
"""
|
|
self._manager = manager
|
|
self.context = context
|
|
self.path = path
|
|
self.checksum = checksum
|
|
self.timestamp = timestamp
|
|
def open(self, mode='r'):
|
|
"""Open the file.
|
|
|
|
This should always be called as as part of a with statement. When
|
|
writing the content is not saved on the server until leaving the with
|
|
statement normally, if an error occurs the server is left unchanged.
|
|
|
|
Example:
|
|
with locFile.open('w') as output:
|
|
output.write('some content')
|
|
|
|
Args:
|
|
mode: 'r' for reading the file, 'w' for writing
|
|
|
|
Returns:
|
|
A file like object that can be used for reads or writes.
|
|
"""
|
|
if mode == 'r':
|
|
request = self._manager._buildRequest(self.context, self.path)
|
|
response = urllib2.urlopen(request)
|
|
# Not the recommended way of reading directories.
|
|
if not(self.isDirectory()):
|
|
checksum = response.headers["Content-MD5"]
|
|
if self.checksum != checksum:
|
|
raise RuntimeError("Localization checksum mismatch " + self.checksum + " " + checksum)
|
|
return contextlib.closing(response)
|
|
elif mode == 'w':
|
|
return _LocalizationOutput(self._manager, self)
|
|
else:
|
|
raise ValueError("mode string must be 'r' or 'w' not " + str(r))
|
|
def delete(self):
|
|
"""Delete this file from the server"""
|
|
request = self._manager._buildRequest(self.context, self.path, method='DELETE')
|
|
request.add_header("If-Match", self.checksum)
|
|
try:
|
|
urllib2.urlopen(request)
|
|
except urllib2.HTTPError as e:
|
|
if e.code == 409:
|
|
raise LocalizationFileVersionConflictException(e.read())
|
|
else:
|
|
raise e
|
|
def exists(self):
|
|
"""Check if this file actually exists.
|
|
|
|
Returns:
|
|
boolean indicating existence of this file
|
|
"""
|
|
return self.checksum != NON_EXISTENT_CHECKSUM
|
|
def isDirectory(self):
|
|
"""Check if this file is a directory.
|
|
|
|
A file must exist to be considered a directory.
|
|
|
|
Returns:
|
|
boolean indicating directorocity of this file
|
|
"""
|
|
return self.checksum == DIRECTORY_CHECKSUM
|
|
def getCheckSum(self):
|
|
return self.checksum
|
|
def getContext(self):
|
|
return self.context
|
|
def getPath(self):
|
|
return self.path
|
|
def getTimeStamp(self):
|
|
return self.timestamp
|
|
def __str__(self):
|
|
return str(self.context) + "/" + self.path
|
|
def __eq__(self, other):
|
|
return self.context == other.context and \
|
|
self.path == other.path and \
|
|
self.checksum == other.checksum \
|
|
and self.timestamp == other.timestamp
|
|
def __hash__(self):
|
|
return hash((self.context, self.path, self.checksum, self.timestamp))
|
|
|
|
def _getHost():
|
|
import subprocess
|
|
host = subprocess.check_output(
|
|
"source /awips2/fxa/bin/setup.env; echo $DEFAULT_HOST",
|
|
shell=True).strip()
|
|
if host:
|
|
return host
|
|
return 'localhost'
|
|
|
|
def _getSiteFromServer(host):
|
|
try:
|
|
from awips import ThriftClient
|
|
from dynamicserialize.dstypes.com.raytheon.uf.common.site.requests import GetPrimarySiteRequest
|
|
client = ThriftClient.ThriftClient(host)
|
|
return client.sendRequest(GetPrimarySiteRequest())
|
|
except:
|
|
# Servers that don't have GFE installed will not return a site
|
|
pass
|
|
|
|
def _getSiteFromEnv():
|
|
site = os.environ.get('FXA_LOCAL_SITE')
|
|
if site is None:
|
|
site = os.environ.get('SITE_IDENTIFIER');
|
|
return site
|
|
|
|
def _getSite(host):
|
|
site = _getSiteFromEnv()
|
|
if not(site):
|
|
site = _getSiteFromServer(host)
|
|
return site
|
|
|
|
def _parseJsonList(manager, response, context, path):
|
|
fileList = []
|
|
jsonResponse = loadjson(response)
|
|
for name, jsonData in jsonResponse.items():
|
|
checksum = jsonData["checksum"]
|
|
timestampString = jsonData["timestamp"]
|
|
timestamp = dateutil.parser.parse(timestampString)
|
|
newpath = urljoin(path, name)
|
|
fileList.append(LocalizationFile(manager, context, newpath, checksum, timestamp))
|
|
return fileList
|
|
|
|
def _parseXmlList(manager, response, context, path):
|
|
fileList = []
|
|
for xmlData in parseXml(response).getroot().findall('file'):
|
|
name = xmlData.get("name")
|
|
checksum = xmlData.get("checksum")
|
|
timestampString = xmlData.get("timestamp")
|
|
timestamp = dateutil.parser.parse(timestampString)
|
|
newpath = urljoin(path, name)
|
|
fileList.append(LocalizationFile(manager, context, newpath, checksum, timestamp))
|
|
return fileList
|
|
|
|
class LocalizationFileManager(object):
|
|
"""Connects to a server and retrieves LocalizationFiles."""
|
|
def __init__(self, host=None, port=9581, path="/services/localization/", contexts=None, site=None, type="common_static"):
|
|
"""Initializes a LocalizationFileManager with connection parameters and context information
|
|
|
|
All arguments are optional and will use defaults or attempt to figure out appropriate values form the environment.
|
|
|
|
Args:
|
|
host: A hostname of the localization server, such as 'ec'.
|
|
port: A port to use to connect to the localization server, usually 9581.
|
|
path: A path to reach the localization file service on the server.
|
|
contexts: A list of contexts to check for files, the order of the contexts will be used
|
|
for the order of incremental results and the priority of absolute results.
|
|
site: A site identifier to use for site specific contexts. This is only used if the contexts arg is None.
|
|
type: A localization type for contexts. This is only used if the contexts arg is None.
|
|
|
|
"""
|
|
if host is None:
|
|
host = _getHost()
|
|
if contexts is None:
|
|
if site is None :
|
|
site = _getSite(host)
|
|
contexts = [LocalizationContext("base", None, type)]
|
|
if site:
|
|
contexts.append(LocalizationContext("configured", site, type))
|
|
contexts.append(LocalizationContext("site", site, type))
|
|
contexts.append(LocalizationContext("user", getuser(), type))
|
|
netloc = host + ':' + str(port)
|
|
self._baseUrl = urlunparse(('http', netloc, path, None, None, None))
|
|
self._contexts = contexts
|
|
def _buildRequest(self, context, path, method='GET'):
|
|
url = urljoin(self._baseUrl, context._getUrlComponent())
|
|
url = urljoin(url, path)
|
|
request = urllib2.Request(url)
|
|
username = getuser()
|
|
# Currently password is ignored in the server
|
|
# this is the defacto standard for not providing one to this service.
|
|
password = username
|
|
base64string = b64encode('%s:%s' % (username, password))
|
|
request.add_header("Authorization", "Basic %s" % base64string)
|
|
if method != 'GET':
|
|
request.get_method = lambda: method
|
|
return request
|
|
def _normalizePath(self, path):
|
|
if path == '' or path == '/':
|
|
path = '.'
|
|
if path[0] == '/':
|
|
path = path[1:]
|
|
return path
|
|
def _list(self, path):
|
|
path = self._normalizePath(path)
|
|
if path[-1] != '/':
|
|
path += '/'
|
|
fileList = []
|
|
exists = False
|
|
for context in self._contexts:
|
|
try:
|
|
request = self._buildRequest(context, path)
|
|
request.add_header("Accept", "application/json, application/xml")
|
|
response = urllib2.urlopen(request)
|
|
exists = True
|
|
if not(response.geturl().endswith("/")):
|
|
# For ordinary files the server sends a redirect to remove the slash.
|
|
raise LocalizationFileIsNotDirectoryException("Not a directory: " + path)
|
|
elif response.headers["Content-Type"] == "application/xml":
|
|
fileList += _parseXmlList(self, response, context, path)
|
|
else:
|
|
fileList += _parseJsonList(self, response, context, path)
|
|
except urllib2.HTTPError as e:
|
|
if e.code != 404:
|
|
raise e
|
|
if not(exists):
|
|
raise LocalizationFileDoesNotExistException("No such file or directory: " + path)
|
|
return fileList
|
|
def _get(self, context, path):
|
|
path = self._normalizePath(path)
|
|
try:
|
|
request = self._buildRequest(context, path, method='HEAD')
|
|
resp = urllib2.urlopen(request)
|
|
if (resp.geturl().endswith("/")):
|
|
checksum = DIRECTORY_CHECKSUM;
|
|
else:
|
|
if "Content-MD5" not in resp.headers:
|
|
raise RuntimeError("Missing Content-MD5 header in response from " + resp.geturl())
|
|
checksum = resp.headers["Content-MD5"]
|
|
if "Last-Modified" not in resp.headers:
|
|
raise RuntimeError("Missing Last-Modified header in response from " + resp.geturl())
|
|
timestamp = dateutil.parser.parse(resp.headers["Last-Modified"])
|
|
return LocalizationFile(self, context, path, checksum, timestamp)
|
|
except urllib2.HTTPError as e:
|
|
if e.code != 404:
|
|
raise e
|
|
else:
|
|
return LocalizationFile(self, context, path, NON_EXISTENT_CHECKSUM, None)
|
|
def listAbsolute(self, path):
|
|
"""List the files in a localization directory, only a single file is returned for each unique path.
|
|
|
|
If a file exists in more than one context then the highest level(furthest from base) is used.
|
|
|
|
Args:
|
|
path: A path to a directory that should be the root of the listing
|
|
|
|
Returns:
|
|
A list of LocalizationFiles
|
|
"""
|
|
merged = dict()
|
|
for file in self._list(path):
|
|
merged[file.path] = file
|
|
return sorted(merged.values(), key=lambda file: file.path)
|
|
def listIncremental(self, path):
|
|
"""List the files in a localization directory, this includes all files for all contexts.
|
|
|
|
Args:
|
|
path: A path to a directory that should be the root of the listing
|
|
|
|
Returns:
|
|
A list of tuples, each tuple will contain one or more files for the
|
|
same paths but different contexts. Each tuple will be ordered the
|
|
same as the contexts in this manager, generally with 'base' first
|
|
and 'user' last.
|
|
"""
|
|
merged = dict()
|
|
for file in self._list(path):
|
|
if file.path in merged:
|
|
merged[file.path] += (file,)
|
|
else:
|
|
merged[file.path] = (file, )
|
|
return sorted(merged.values(), key=lambda t: t[0].path)
|
|
def getAbsolute(self, path):
|
|
"""Get a single localization file from the highest level context where it exists.
|
|
|
|
Args:
|
|
path: A path to a localization file
|
|
|
|
Returns:
|
|
A Localization File with the specified path or None if the file does not exist in any context.
|
|
|
|
"""
|
|
for context in reversed(self._contexts):
|
|
f = self._get(context, path)
|
|
if f.exists():
|
|
return f
|
|
def getIncremental(self, path):
|
|
"""Get all the localization files that exist in any context for the provided path.
|
|
|
|
Args:
|
|
path: A path to a localization file
|
|
|
|
Returns:
|
|
A tuple containing all the files that exist for this path in any context. The tuple
|
|
will be ordered the same as the contexts in this manager, generally with 'base' first
|
|
and 'user' last.
|
|
"""
|
|
result = ()
|
|
for context in self._contexts:
|
|
f = self._get(context, path)
|
|
if f.exists():
|
|
result += (f,)
|
|
return result
|
|
def getSpecific(self, level, path):
|
|
"""Get a specific localization file at a given level, the file may not exist.
|
|
|
|
The file is returned for whichever context is valid for the provided level in this manager.
|
|
|
|
For writing new files this is the only way to get access to a file that
|
|
does not exist in order to create it.
|
|
|
|
Args:
|
|
level: the name of a localization level, such as "base", "site", "user"
|
|
path: A path to a localization file
|
|
|
|
Returns:
|
|
A Localization File with the specified path and a context for the specified level.
|
|
"""
|
|
for context in self._contexts:
|
|
if context.level == level:
|
|
return self._get(context, path)
|
|
raise ValueError("No context defined for level " + level)
|
|
def __str__(self):
|
|
contextsStr = '[' + ' '.join((str(c) for c in self._contexts)) + ']'
|
|
return '<' + self.__class__.__name__ + " for " + self._baseUrl + ' ' + contextsStr + '>'
|