xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #1230: Add logging of long running hdf5 requests" into development

Browse source 
Former-commit-id: ff5780cbed [formerly ff5780cbed [formerly 7d2f033a4e994e655d0839af0c2cb91543c5f55c]]
Former-commit-id: ec9c804d5b
Former-commit-id: 0e055090dd
This commit is contained in:
Nate Jensen 2012-10-02 09:49:56 -05:00 • committed by Gerrit Code Review
commit c9b2083dd5
2 changed files with 55 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

84
edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/PyPiesDataStore.java
View file

@ -63,7 +63,7 @@ import com.raytheon.uf.common.util.FileUtil;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* May 27, 2010 njensen Initial creation * May 27, 2010 njensen Initial creation
* * Oct 01, 2010 rjpeter Added logging of requests over 300ms
* </pre> * </pre>
* *
* @author njensen * @author njensen
@ -72,6 +72,8 @@ import com.raytheon.uf.common.util.FileUtil;
public class PyPiesDataStore implements IDataStore { public class PyPiesDataStore implements IDataStore {
private static final long SIMPLE_LOG_TIME = 300;
protected static String address = null; protected static String address = null;
protected List<IDataRecord> records = new ArrayList<IDataRecord>(); protected List<IDataRecord> records = new ArrayList<IDataRecord>();
@ -80,7 +82,8 @@ public class PyPiesDataStore implements IDataStore {
protected PypiesProperties props; protected PypiesProperties props;
public PyPiesDataStore(File file, boolean useLocking, PypiesProperties props) { public PyPiesDataStore(final File file, final boolean useLocking,
final PypiesProperties props) {
this.filename = FileUtil.edexPath(file.getPath()); // Win32 this.filename = FileUtil.edexPath(file.getPath()); // Win32
this.props = props; this.props = props;
} }
@ -94,8 +97,8 @@ public class PyPiesDataStore implements IDataStore {
* com.raytheon.uf.common.datastorage.StorageProperties) * com.raytheon.uf.common.datastorage.StorageProperties)
*/ */
@Override @Override
public void addDataRecord(IDataRecord dataset, StorageProperties properties) public void addDataRecord(final IDataRecord dataset,
throws StorageException { final StorageProperties properties) throws StorageException {
if (dataset.validateDataSet()) { if (dataset.validateDataSet()) {
dataset.setProperties(properties); dataset.setProperties(properties);
records.add(dataset); records.add(dataset);
@ -114,7 +117,8 @@ public class PyPiesDataStore implements IDataStore {
* .uf.common.datastorage.records.IDataRecord) * .uf.common.datastorage.records.IDataRecord)
*/ */
@Override @Override
public void addDataRecord(IDataRecord dataset) throws StorageException { public void addDataRecord(final IDataRecord dataset)
throws StorageException {
addDataRecord(dataset, dataset.getProperties()); addDataRecord(dataset, dataset.getProperties());
} }
@ -125,7 +129,7 @@ public class PyPiesDataStore implements IDataStore {
* com.raytheon.uf.common.datastorage.IDataStore#createLinks(java.util.Map) * com.raytheon.uf.common.datastorage.IDataStore#createLinks(java.util.Map)
*/ */
@Override @Override
public void createLinks(Map<String, LinkLocation> links) public void createLinks(final Map<String, LinkLocation> links)
throws StorageException, FileNotFoundException { throws StorageException, FileNotFoundException {
throw new UnsupportedOperationException( throw new UnsupportedOperationException(
"pypies does not support this yet!"); "pypies does not support this yet!");
@ -138,7 +142,7 @@ public class PyPiesDataStore implements IDataStore {
* com.raytheon.uf.common.datastorage.IDataStore#delete(java.lang.String[]) * com.raytheon.uf.common.datastorage.IDataStore#delete(java.lang.String[])
*/ */
@Override @Override
public void delete(String... location) throws StorageException, public void delete(final String... location) throws StorageException,
FileNotFoundException { FileNotFoundException {
DeleteRequest delete = new DeleteRequest(); DeleteRequest delete = new DeleteRequest();
delete.setLocations(location); delete.setLocations(location);
@ -153,7 +157,7 @@ public class PyPiesDataStore implements IDataStore {
* ) * )
*/ */
@Override @Override
public String[] getDatasets(String group) throws StorageException, public String[] getDatasets(final String group) throws StorageException,
FileNotFoundException { FileNotFoundException {
DatasetNamesRequest req = new DatasetNamesRequest(); DatasetNamesRequest req = new DatasetNamesRequest();
req.setGroup(group); req.setGroup(group);
@ -168,7 +172,7 @@ public class PyPiesDataStore implements IDataStore {
* com.raytheon.uf.common.datastorage.IDataStore#retrieve(java.lang.String) * com.raytheon.uf.common.datastorage.IDataStore#retrieve(java.lang.String)
*/ */
@Override @Override
public IDataRecord[] retrieve(String group) throws StorageException, public IDataRecord[] retrieve(final String group) throws StorageException,
FileNotFoundException { FileNotFoundException {
return retrieve(group, false); return retrieve(group, false);
} }
@ -181,8 +185,9 @@ public class PyPiesDataStore implements IDataStore {
* boolean) * boolean)
*/ */
@Override @Override
public IDataRecord[] retrieve(String group, boolean includeInterpolated) public IDataRecord[] retrieve(final String group,
throws StorageException, FileNotFoundException { final boolean includeInterpolated) throws StorageException,
FileNotFoundException {
RetrieveRequest req = new RetrieveRequest(); RetrieveRequest req = new RetrieveRequest();
req.setGroup(group); req.setGroup(group);
req.setIncludeInterpolated(includeInterpolated); req.setIncludeInterpolated(includeInterpolated);
@ -198,8 +203,9 @@ public class PyPiesDataStore implements IDataStore {
* java.lang.String, com.raytheon.uf.common.datastorage.Request) * java.lang.String, com.raytheon.uf.common.datastorage.Request)
*/ */
@Override @Override
public IDataRecord retrieve(String group, String dataset, Request request) public IDataRecord retrieve(final String group, final String dataset,
throws StorageException, FileNotFoundException { final Request request) throws StorageException,
FileNotFoundException {
RetrieveRequest req = new RetrieveRequest(); RetrieveRequest req = new RetrieveRequest();
req.setGroup(group); req.setGroup(group);
req.setDataset(dataset); req.setDataset(dataset);
@ -216,8 +222,9 @@ public class PyPiesDataStore implements IDataStore {
* .String[], com.raytheon.uf.common.datastorage.Request) * .String[], com.raytheon.uf.common.datastorage.Request)
*/ */
@Override @Override
public IDataRecord[] retrieveDatasets(String[] datasetGroupPath, public IDataRecord[] retrieveDatasets(final String[] datasetGroupPath,
Request request) throws StorageException, FileNotFoundException { final Request request) throws StorageException,
FileNotFoundException {
DatasetDataRequest req = new DatasetDataRequest(); DatasetDataRequest req = new DatasetDataRequest();
req.setDatasetGroupPath(datasetGroupPath); req.setDatasetGroupPath(datasetGroupPath);
req.setRequest(request); req.setRequest(request);
@ -233,8 +240,9 @@ public class PyPiesDataStore implements IDataStore {
* .String[], com.raytheon.uf.common.datastorage.Request) * .String[], com.raytheon.uf.common.datastorage.Request)
*/ */
@Override @Override
public IDataRecord[] retrieveGroups(String[] groups, Request request) public IDataRecord[] retrieveGroups(final String[] groups,
throws StorageException, FileNotFoundException { final Request request) throws StorageException,
FileNotFoundException {
GroupsRequest req = new GroupsRequest(); GroupsRequest req = new GroupsRequest();
req.setGroups(groups); req.setGroups(groups);
req.setRequest(request); req.setRequest(request);
@ -261,7 +269,7 @@ public class PyPiesDataStore implements IDataStore {
* .datastorage.IDataStore.StoreOp) * .datastorage.IDataStore.StoreOp)
*/ */
@Override @Override
public StorageStatus store(StoreOp storeOp) throws StorageException { public StorageStatus store(final StoreOp storeOp) throws StorageException {
StoreRequest req = new StoreRequest(); StoreRequest req = new StoreRequest();
req.setOp(storeOp); req.setOp(storeOp);
req.setRecords(records); req.setRecords(records);
@ -293,19 +301,28 @@ public class PyPiesDataStore implements IDataStore {
return ss; return ss;
} }
protected Object sendRequest(AbstractRequest obj) throws StorageException { protected Object sendRequest(final AbstractRequest obj)
throws StorageException {
obj.setFilename(filename); obj.setFilename(filename);
byte[] bytes = serializeRequest(obj); byte[] bytes = serializeRequest(obj);
initializeProperties(); initializeProperties();
byte[] result = null; byte[] result = null;
long t0 = System.currentTimeMillis();
try { try {
result = HttpClient.getInstance().postBinary(address, bytes); result = HttpClient.getInstance().postBinary(address, bytes);
} catch (Exception e) { } catch (Exception e) {
throw new StorageException( throw new StorageException(
"Error communicating with pypies server", null, e); "Error communicating with pypies server", null, e);
} }
long time = System.currentTimeMillis() - t0;
if (time >= SIMPLE_LOG_TIME) {
System.out.println("Took " + time + " ms to receive response for "
+ obj.getClass().getSimpleName() + " on file "
+ obj.getFilename());
}
Object ret = deserializeResponse(result); Object ret = deserializeResponse(result);
@ -325,11 +342,12 @@ public class PyPiesDataStore implements IDataStore {
* @return * @return
* @throws StorageException * @throws StorageException
*/ */
protected Object cachedRequest(AbstractRequest obj) throws StorageException { protected Object cachedRequest(final AbstractRequest obj)
throws StorageException {
return this.sendRequest(obj); return this.sendRequest(obj);
} }
protected byte[] serializeRequest(AbstractRequest request) protected byte[] serializeRequest(final AbstractRequest request)
throws StorageException { throws StorageException {
try { try {
return SerializationUtil.transformToThrift(request); return SerializationUtil.transformToThrift(request);
@ -338,7 +356,7 @@ public class PyPiesDataStore implements IDataStore {
} }
} }
protected Object deserializeResponse(byte[] response) protected Object deserializeResponse(final byte[] response)
throws StorageException { throws StorageException {
try { try {
return SerializationUtil.transformFromThrift(response); return SerializationUtil.transformFromThrift(response);
@ -359,15 +377,15 @@ public class PyPiesDataStore implements IDataStore {
} }
@Override @Override
public void deleteFiles(String[] datesToDelete) throws StorageException, public void deleteFiles(final String[] datesToDelete)
FileNotFoundException { throws StorageException, FileNotFoundException {
DeleteFilesRequest req = new DeleteFilesRequest(); DeleteFilesRequest req = new DeleteFilesRequest();
req.setDatesToDelete(datesToDelete); req.setDatesToDelete(datesToDelete);
sendRequest(req); sendRequest(req);
} }
@Override @Override
public void createDataset(IDataRecord rec) throws StorageException, public void createDataset(final IDataRecord rec) throws StorageException,
FileNotFoundException { FileNotFoundException {
CreateDatasetRequest req = new CreateDatasetRequest(); CreateDatasetRequest req = new CreateDatasetRequest();
req.setRecord(rec); req.setRecord(rec);
@ -375,7 +393,7 @@ public class PyPiesDataStore implements IDataStore {
} }
@Override @Override
public void repack(Compression compression) throws StorageException { public void repack(final Compression compression) throws StorageException {
RepackRequest req = new RepackRequest(); RepackRequest req = new RepackRequest();
req.setFilename(this.filename); req.setFilename(this.filename);
req.setCompression(compression); req.setCompression(compression);
@ -383,8 +401,8 @@ public class PyPiesDataStore implements IDataStore {
// TODO do we really want to make this an exception? // TODO do we really want to make this an exception?
// reasoning is if the repack fails for some reason, the original file // reasoning is if the repack fails for some reason, the original file
// is left as is, just isn't as efficiently packed // is left as is, just isn't as efficiently packed
if (resp != null && resp.getFailedFiles() != null if ((resp != null) && (resp.getFailedFiles() != null)
&& resp.getFailedFiles().length > 0) { && (resp.getFailedFiles().length > 0)) {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append("Error repacking the following files: "); sb.append("Error repacking the following files: ");
String[] failed = resp.getFailedFiles(); String[] failed = resp.getFailedFiles();
@ -399,9 +417,9 @@ public class PyPiesDataStore implements IDataStore {
} }
@Override @Override
public void copy(String outputDir, Compression compression, public void copy(final String outputDir, final Compression compression,
String timestampCheck, int minMillisSinceLastChange, final String timestampCheck, final int minMillisSinceLastChange,
int maxMillisSinceLastChange) throws StorageException { final int maxMillisSinceLastChange) throws StorageException {
CopyRequest req = new CopyRequest(); CopyRequest req = new CopyRequest();
req.setFilename(this.filename); req.setFilename(this.filename);
if (compression != null) { if (compression != null) {
@ -415,8 +433,8 @@ public class PyPiesDataStore implements IDataStore {
req.setMinMillisSinceLastChange(minMillisSinceLastChange); req.setMinMillisSinceLastChange(minMillisSinceLastChange);
FileActionResponse resp = (FileActionResponse) sendRequest(req); FileActionResponse resp = (FileActionResponse) sendRequest(req);
if (resp != null && resp.getFailedFiles() != null if ((resp != null) && (resp.getFailedFiles() != null)
&& resp.getFailedFiles().length > 0) { && (resp.getFailedFiles().length > 0)) {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append("Error copying the following files: "); sb.append("Error copying the following files: ");
String[] failed = resp.getFailedFiles(); String[] failed = resp.getFailedFiles();

8
pythonPackages/pypies/pypies/impl/H5pyDataStore.py
View file

@ -340,10 +340,7 @@ class H5pyDataStore(IDataStore.IDataStore):
group = request.getGroup() group = request.getGroup()
req = request.getRequest() req = request.getRequest()
if req: if req:
t0=time.time()
grp = self.__getGroup(f, group) grp = self.__getGroup(f, group)
t1=time.time()
timeMap['getGroup']=t1-t0
result = [self.__retrieveInternal(grp, request.getDataset(), req)] result = [self.__retrieveInternal(grp, request.getDataset(), req)]
else: else:
result = self.__retrieve(f, group, request.getIncludeInterpolated()) result = self.__retrieve(f, group, request.getIncludeInterpolated())
@ -711,7 +708,10 @@ class H5pyDataStore(IDataStore.IDataStore):
# directory, so just copy the original data without the repack # directory, so just copy the original data without the repack
shutil.copy(filepath, repackedFullPath) shutil.copy(filepath, repackedFullPath)
t1=time.time() t1=time.time()
timeMap['repack'] if timeMap.has_key('repack'):
timeMap['repack']+=t1-t0
else:
timeMap['repack']=t1-t0
return success return success
def __doFileAction(self, filepath, basePath, outputDir, fileAction, response, compression='NONE', timestampCheck=None): def __doFileAction(self, filepath, basePath, outputDir, fileAction, response, compression='NONE', timestampCheck=None):