xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Omaha #3149 Limit number of grids returned by GFE to constrain size of Thrift response" into omaha_15.1.1

Browse source 
Former-commit-id: 06a1d1555c [formerly 4312c9c569 [formerly 42ec8874c8] [formerly 06a1d1555c [formerly 6d960e34632c2f615720c0550296d38feb8c90f6]]]
Former-commit-id: 4312c9c569 [formerly 42ec8874c8]
Former-commit-id: 4312c9c569
Former-commit-id: c2deb63b76
This commit is contained in:
Richard Peter 2014-07-07 09:30:09 -05:00 • committed by Gerrit Code Review
commit a9444af511
11 changed files with 152 additions and 287 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/core/internal/IFPClient.java
View file

@ -120,6 +120,8 @@ import com.raytheon.viz.gfe.core.parm.Parm;
* 11/20/2013 #2331 randerso Added getTopoData method
* 04/03/2014 #2737 randerso Moved clientISCSendStatus to SaveGFEGridRequest
* 04/09/2014 #3004 dgilling Support moved ClearPracticeVTECTableRequest.
* 07/01/2014 #3149 randerso Changed getGridData to handle limited number of grids returned
* and re-request if not all data returned
*
* </pre>
*
@ -563,21 +565,34 @@ public class IFPClient {
* @return List of grid slices
* @throws GFEServerException
*/
@SuppressWarnings("unchecked")
public List<IGridSlice> getGridData(ParmID parmId, List<TimeRange> gridTimes)
throws GFEServerException {
return getGridData(parmId, gridTimes, false);
}
@SuppressWarnings("unchecked")
public List<IGridSlice> getGridData(ParmID parmId,
List<TimeRange> gridTimes, boolean convertUnit)
throws GFEServerException {
GetGridRequest req = new GetGridRequest(parmId, gridTimes);
req.setConvertUnit(convertUnit);
GetGridDataRequest request = new GetGridDataRequest();
request.addRequest(req);
ServerResponse<?> resp = makeRequest(request);
List<IGridSlice> slices = (List<IGridSlice>) resp.getPayload();
List<IGridSlice> slices = new ArrayList<IGridSlice>(gridTimes.size());
while (slices.size() < gridTimes.size()) {
ServerResponse<List<IGridSlice>> resp = (ServerResponse<List<IGridSlice>>) makeRequest(request);
slices.addAll(resp.getPayload());
// if no slices returned (shouldn't happen unless server code is
// broken)
if (slices.isEmpty()) {
String msg = "No data returned from GetGridDataRequest for "
+ parmId + " for times:" + req.getTimes();
statusHandler.error(msg);
throw new GFEServerException(msg);
}
// if not all slices returned
if (slices.size() < gridTimes.size()) {
// request remaining times.
req.setTimes(gridTimes.subList(slices.size(), gridTimes.size()));
}
}
return slices;
}

9
edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-request.xml
View file

@ -52,16 +52,13 @@
<constructor-arg value="com.raytheon.uf.common.dataplugin.gfe.request.GetDiscreteDefinitionRequest"/>
<constructor-arg ref="discreteDefinitionHandler"/>
</bean>
<bean id="gridDataHandler" class="com.raytheon.edex.plugin.gfe.server.handler.GetGridDataHandler"/>
<bean id="gridDataHandler" class="com.raytheon.edex.plugin.gfe.server.handler.GetGridDataHandler">
<property name="byteLimitInMB" value="${thriftService.byteLimitInMB}" />
</bean>
<bean factory-bean="handlerRegistry" factory-method="register">
<constructor-arg value="com.raytheon.uf.common.dataplugin.gfe.request.GetGridDataRequest"/>
<constructor-arg ref="gridDataHandler"/>
</bean>
<bean id="pythonGridDataHandler" class="com.raytheon.edex.plugin.gfe.server.handler.GetPythonGridDataHandler"/>
<bean factory-bean="handlerRegistry" factory-method="register">
<constructor-arg value="com.raytheon.uf.common.dataplugin.gfe.request.GetPythonGridDataRequest"/>
<constructor-arg ref="pythonGridDataHandler"/>
</bean>
<bean id="gridInventoryHandler" class="com.raytheon.edex.plugin.gfe.server.handler.GetGridInventoryHandler"/>
<bean id="getGridInventoryRequestRegister" factory-bean="handlerRegistry" factory-method="register">
<constructor-arg value="com.raytheon.uf.common.dataplugin.gfe.request.GetGridInventoryRequest"/>

9
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParm.java
View file

@ -28,7 +28,6 @@ import java.util.List;
import java.util.Map;
import com.raytheon.edex.plugin.gfe.db.dao.GFEDao;
import com.raytheon.edex.plugin.gfe.server.database.D2DGridDatabase;
import com.raytheon.edex.plugin.gfe.server.database.GridDatabase;
import com.raytheon.edex.plugin.gfe.server.lock.LockManager;
import com.raytheon.uf.common.dataplugin.gfe.GridDataHistory;
@ -76,6 +75,7 @@ import com.raytheon.uf.edex.database.plugin.PluginFactory;
* 04/23/13 #1949 rjpeter Removed excess validation on retrieval, added
* inventory for a given time range.
* 06/13/13 #2044 randerso Refactored to use non-singleton LockManager
* 07/01/2014 #3149 randerso Removed unit conversion (unused)
* </pre>
*
* @author bphillip
@ -325,12 +325,7 @@ public class GridParm {
if (!CollectionUtil.isNullOrEmpty(reqTimes)) {
// Get the data
if (getRequest.isConvertUnit() && (db instanceof D2DGridDatabase)) {
sr = ((D2DGridDatabase) db).getGridData(id, reqTimes,
getRequest.isConvertUnit());
} else {
sr = db.getGridData(id, reqTimes);
}
sr = db.getGridData(id, reqTimes);
if (!sr.isOkay()) {
sr.addMessage("Failure in retrieving grid data from GridDatabase");
return sr;

106
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/GetGridDataHandler.java
View file

@ -20,23 +20,35 @@
package com.raytheon.edex.plugin.gfe.server.handler;
import java.util.Arrays;
import java.util.List;
import com.raytheon.edex.plugin.gfe.server.GridParmManager;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.GridParmInfo;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.ParmID;
import com.raytheon.uf.common.dataplugin.gfe.request.GetGridDataRequest;
import com.raytheon.uf.common.dataplugin.gfe.server.message.ServerResponse;
import com.raytheon.uf.common.dataplugin.gfe.server.request.GetGridRequest;
import com.raytheon.uf.common.dataplugin.gfe.slice.IGridSlice;
import com.raytheon.uf.common.serialization.comm.IRequestHandler;
import com.raytheon.uf.common.time.TimeRange;
import com.raytheon.uf.common.util.SizeUtil;
/**
* GFE task for getting grid data slices
*
* May return less than the full amount of data requested if returned grids
* exceed MAX_BYTES_PER_REQUEST in total size. The requestor is expected to
* re-request remaining data.
*
* <pre>
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 04/18/08 #875 bphillip Initial Creation
* 09/22/09 3058 rjpeter Converted to IRequestHandler
* 06/13/13 2044 randerso Refactored to use IFPServer
* 06/13/13 2044 randerso Refactored to use IFPServer
* 07/01/2014 #3149 randerso Changed to limit size of data returned
* </pre>
*
* @author randerso
@ -44,10 +56,98 @@ import com.raytheon.uf.common.serialization.comm.IRequestHandler;
*/
public class GetGridDataHandler extends BaseGfeRequestHandler implements
IRequestHandler<GetGridDataRequest> {
private int byteLimitInMB = 100;
/**
* @return the byteLimitInMB
*/
public int getByteLimitInMB() {
return this.byteLimitInMB;
}
/**
* @param byteLimitInMB
* the byteLimitInMB to set
*/
public void setByteLimitInMB(int byteLimitInMB) {
this.byteLimitInMB = byteLimitInMB;
}
@Override
public ServerResponse<List<IGridSlice>> handleRequest(
GetGridDataRequest request) throws Exception {
return getIfpServer(request).getGridParmMgr().getGridData(
request.getRequests());
ServerResponse<List<IGridSlice>> sr = new ServerResponse<List<IGridSlice>>();
GridParmManager gpm = getIfpServer(request).getGridParmMgr();
int gridCount = 0;
int remaining = (int) (this.byteLimitInMB * SizeUtil.BYTES_PER_MB * 0.9);
List<IGridSlice> data = null;
for (GetGridRequest req : request.getRequests()) {
ParmID parmId = req.getParmId();
List<TimeRange> times = req.getTimes();
ServerResponse<GridParmInfo> ss1 = gpm.getGridParmInfo(parmId);
if (!ss1.isOkay()) {
sr.addMessages(ss1);
return sr;
}
GridParmInfo gpi = ss1.getPayload();
int gridSize = gpi.getGridLoc().getNx() * gpi.getGridLoc().getNy();
switch (gpi.getGridType()) {
case SCALAR:
gridSize *= 4;
break;
case VECTOR:
gridSize *= 8;
break;
case WEATHER:
case DISCRETE:
break;
default:
break;
}
int maxGrids = remaining / gridSize;
// ensure we return at least 1 grid
if ((maxGrids == 0) && (gridCount == 0)) {
maxGrids = 1;
}
// no more grids will fit break out of loop
if (maxGrids == 0) {
break;
}
if (maxGrids < times.size()) {
// truncate the request
times = times.subList(0, maxGrids);
req.setTimes(times);
}
gridCount += times.size();
remaining -= times.size() * gridSize;
ServerResponse<List<IGridSlice>> ssr = gpm.getGridData(Arrays
.asList(req));
if (ssr.isOkay()) {
if (data == null) {
data = ssr.getPayload();
} else {
data.addAll(ssr.getPayload());
}
} else {
sr.addMessages(ssr);
break;
}
}
if (sr.isOkay()) {
sr.setPayload(data);
}
return sr;
}
}

94
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/GetPythonGridDataHandler.java
View file

@ -1,94 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.edex.plugin.gfe.server.handler;
import java.util.ArrayList;
import java.util.List;
import com.raytheon.uf.common.dataplugin.gfe.request.GetPythonGridDataRequest;
import com.raytheon.uf.common.dataplugin.gfe.server.message.ServerResponse;
import com.raytheon.uf.common.dataplugin.gfe.slice.IGridSlice;
import com.raytheon.uf.common.dataplugin.gfe.slice.PythonWeatherGridSlice;
import com.raytheon.uf.common.dataplugin.gfe.slice.WeatherGridSlice;
import com.raytheon.uf.common.serialization.comm.IRequestHandler;
/**
* Handler for GetPythonGridDataRequest
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 4, 2011 dgilling Initial creation
* Jun 13, 2013 #2044 randerso Refactored to use IFPServer
*
* </pre>
*
* @deprecated use the Data Access Framework
*
* @author dgilling
* @version 1.0
*/
// TODO: REMOVE THIS CLASS AND ITS REQUEST TYPE if
// DiscreteDefinition/DiscreteKey and WxDefinition/WeatherKey class hierarchy is
// ever fully-implemented in Python.
@Deprecated
public class GetPythonGridDataHandler extends BaseGfeRequestHandler implements
IRequestHandler<GetPythonGridDataRequest> {
/*
* (non-Javadoc)
*
* @see
* com.raytheon.uf.common.serialization.comm.IRequestHandler#handleRequest
* (com.raytheon.uf.common.serialization.comm.IServerRequest)
*/
@Override
public ServerResponse<List<IGridSlice>> handleRequest(
GetPythonGridDataRequest request) throws Exception {
ServerResponse<List<IGridSlice>> finalResp = new ServerResponse<List<IGridSlice>>();
ServerResponse<List<IGridSlice>> sr = getIfpServer(request)
.getGridParmMgr().getGridData(request.getRequests());
if (!sr.isOkay()) {
finalResp.addMessages(sr);
finalResp.setPayload(new ArrayList<IGridSlice>(0));
return finalResp;
}
// convert grid slices as needed
List<IGridSlice> slices = new ArrayList<IGridSlice>(sr.getPayload()
.size());
for (IGridSlice slice : sr.getPayload()) {
if (!(slice instanceof WeatherGridSlice)) {
slices.add(slice);
} else {
slices.add(new PythonWeatherGridSlice((WeatherGridSlice) slice));
}
}
finalResp.setPayload(slices);
return finalResp;
}
}

31
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/smartinit/IFPWE.java
View file

@ -89,6 +89,7 @@ import com.raytheon.uf.common.util.Pair;
* Added getKeys(tr) to get grid times overlapping a time range
* Removed caching of inventory as it was not being updated when
* grids were updated/deleted
* Jul 01, 2014 #3149 randerso Changed to use updated GetGridRequest. Cleaned up code
*
* </pre>
*
@ -201,39 +202,19 @@ public class IFPWE {
* @throws GfeException
*/
public IGridSlice getItem(TimeRange timeRange) throws GfeException {
GetGridRequest req = new GetGridRequest();
req.setParmId(parmId);
GFERecord gfeRec = new GFERecord(parmId, timeRange);
ArrayList<GFERecord> gfeList = new ArrayList<GFERecord>();
gfeList.add(gfeRec);
req.setRecords(gfeList);
GetGridRequest req = new GetGridRequest(parmId,
Arrays.asList(timeRange));
ArrayList<GetGridRequest> reqList = new ArrayList<GetGridRequest>();
reqList.add(req);
List<IGridSlice> data = new ArrayList<IGridSlice>();
ServerResponse<List<IGridSlice>> ssr = gridParmMgr.getGridData(reqList);
data = ssr.getPayload();
IGridSlice slice = null;
if ((data == null) || (data.size() == 0)) {
if (!ssr.isOkay()) {
String msg = "Error getting grid data for " + parmId.toString()
+ " at time " + timeRange.toString();
for (ServerMsg smsg : ssr.getMessages()) {
msg += "\n" + smsg.getMessage();
}
+ " at time " + timeRange.toString() + ssr.message();
throw new GfeException(msg);
} else if (data.size() > 1) {
// theoretically should never get here
String msg = "Retrieved too much data for " + parmId.toString()
+ "at time " + timeRange.toString();
for (ServerMsg smsg : ssr.getMessages()) {
msg += "\n" + smsg.getMessage();
}
throw new GfeException(msg);
} else {
slice = data.get(0);
}
IGridSlice slice = ssr.getPayload().get(0);
return slice;
}

8
edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/dataaccess/GFEDataAccessUtil.java
View file

@ -43,9 +43,10 @@ import com.raytheon.uf.common.serialization.comm.RequestRouter;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 9, 2011 bsteffen Initial creation
* May 02, 2013 1949 bsteffen Update GFE data access in Product
* May 02, 2013 1949 bsteffen Update GFE data access in Product
* Browser, Volume Browser, and Data Access
* Framework.
* Jul 01, 2014 3149 randerso Changed to use updated GetGridRequest
*
* </pre>
*
@ -96,9 +97,8 @@ public class GFEDataAccessUtil {
* @throws Exception
*/
public static IGridSlice getSlice(GFERecord gfeRecord) throws Exception {
GetGridRequest gridRequest = new GetGridRequest();
gridRequest.setParmId(gfeRecord.getParmId());
gridRequest.setRecords(Arrays.asList(gfeRecord));
GetGridRequest gridRequest = new GetGridRequest(gfeRecord.getParmId(),
Arrays.asList(gfeRecord.getDataTime().getValidPeriod()));
GetGridDataRequest request = new GetGridDataRequest();
request.setSiteID(gfeRecord.getDbId().getSiteId());

52
edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/GetPythonGridDataRequest.java
View file

@ -1,52 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.dataplugin.gfe.request;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
/**
* This class is only intended for use with Python clients as it returns
* IGridSlices in a way that can be easily deserialized by Python without
* requiring full implementation the Java class hierarchy.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 4, 2011 dgilling Initial creation
*
* </pre>
*
* @deprecated use the Data Access Framework
*
* @author dgilling
* @version 1.0
*/
// TODO: REMOVE THIS CLASS AND ITS HANDLER if DiscreteDefinition/DiscreteKey and
// WxDefinition/WeatherKey class hierarchy is ever fully-implemented in Python.
@Deprecated
@DynamicSerialize
public class GetPythonGridDataRequest extends GetGridDataRequest {
}

60
edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/server/request/GetGridRequest.java
View file

@ -20,12 +20,9 @@
package com.raytheon.uf.common.dataplugin.gfe.server.request;
import java.util.ArrayList;
import java.util.List;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.GFERecord;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.ParmID;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.raytheon.uf.common.time.TimeRange;
@ -38,6 +35,8 @@ import com.raytheon.uf.common.time.TimeRange;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 04/08/08 #875 bphillip Initial Creation
* 07/01/2014 #3149 randerso Simplified to contain only a ParmID and
* list of TimeRanges.
*
* </pre>
*
@ -46,41 +45,18 @@ import com.raytheon.uf.common.time.TimeRange;
*/
@DynamicSerialize
public class GetGridRequest implements ISerializableObject {
public class GetGridRequest {
/** The records to be saved */
@DynamicSerializeElement
private List<GFERecord> records;
private List<TimeRange> times;
/** The parmID of the request */
@DynamicSerializeElement
private ParmID parmId;
/**
* Denotes whether the data retrieved from this request will be converted to
* match the corresponding grid parm info
*/
@DynamicSerializeElement
private boolean convertUnit = false;
/**
* @return the convertUnit
*/
public boolean isConvertUnit() {
return convertUnit;
}
/**
* @param convertUnit
* the convertUnit to set
*/
public void setConvertUnit(boolean convertUnit) {
this.convertUnit = convertUnit;
}
/**
* Creates a new GetGridRequest
*/
@ -96,19 +72,9 @@ public class GetGridRequest implements ISerializableObject {
* @param times
* The times of the requested grids
*/
public GetGridRequest(List<GFERecord> records) {
if (records.size() > 0) {
parmId = records.get(0).getParmId();
}
this.records = records;
}
public GetGridRequest(ParmID parmId, List<TimeRange> trs) {
public GetGridRequest(ParmID parmId, List<TimeRange> times) {
this.parmId = parmId;
records = new ArrayList<GFERecord>();
for (TimeRange tr : trs) {
records.add(new GFERecord(parmId, tr));
}
this.times = times;
}
public ParmID getParmId() {
@ -120,19 +86,11 @@ public class GetGridRequest implements ISerializableObject {
}
public List<TimeRange> getTimes() {
List<TimeRange> times = new ArrayList<TimeRange>();
for (GFERecord rec : records) {
times.add(rec.getTimeRange());
}
return times;
return this.times;
}
public List<GFERecord> getRecords() {
return records;
}
public void setRecords(List<GFERecord> records) {
this.records = records;
public void setTimes(List<TimeRange> times) {
this.times = times;
}
}

33
pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/GetPythonGridDataRequest.py
View file

@ -1,33 +0,0 @@
##
# This software was developed and / or modified by Raytheon Company,
# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
#
# U.S. EXPORT CONTROLLED TECHNICAL DATA
# This software product contains export-restricted data whose
# export/transfer/disclosure is restricted by U.S. law. Dissemination
# to non-U.S. persons whether in the United States or abroad requires
# an export license or other authorization.
#
# Contractor Name: Raytheon Company
# Contractor Address: 6825 Pine Street, Suite 340
# Mail Stop B8
# Omaha, NE 68106
# 402.291.0100
#
# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
# further licensing information.
##
# File auto-generated against equivalent DynamicSerialize Java class
from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import GetGridDataRequest
class GetPythonGridDataRequest(GetGridDataRequest):
def __init__(self):
super(GetPythonGridDataRequest, self).__init__()
## nothing to implement here that isn't already covered by GetGridDataRequest ##
## Just need the separate class for de-serialization. ##

2
pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/__init__.py
View file

@ -35,7 +35,6 @@ __all__ = [
'GetLockTablesRequest',
'GetOfficialDbNameRequest',
'GetParmListRequest',
'GetPythonGridDataRequest',
'GetSelectTimeRangeRequest',
'GetSingletonDbIdsRequest',
'GetSiteTimeZoneInfoRequest',
@ -64,7 +63,6 @@ from GetLatestModelDbIdRequest import GetLatestModelDbIdRequest
from GetLockTablesRequest import GetLockTablesRequest
from GetOfficialDbNameRequest import GetOfficialDbNameRequest
from GetParmListRequest import GetParmListRequest
from GetPythonGridDataRequest import GetPythonGridDataRequest
from GetSelectTimeRangeRequest import GetSelectTimeRangeRequest
from GetSingletonDbIdsRequest import GetSingletonDbIdsRequest
from GetSiteTimeZoneInfoRequest import GetSiteTimeZoneInfoRequest