xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Omaha #3071 Fix several errors identified by the log tool

Browse source 
Change-Id: I98aa1101fa796bd4ad93d2b8556248bbd13219e4

Former-commit-id: 9ebbc261b1 [formerly 9ebbc261b1 [formerly ca6a36be221da70cd766c6e844e2852bd67f69c3]]
Former-commit-id: ca0946b5f8
Former-commit-id: b45048f5bf
This commit is contained in:
Ron Anderson 2014-06-02 12:08:35 -05:00
parent cbceb4ea62
commit 674baaeabc
3 changed files with 322 additions and 306 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

459
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/GridParmManager.java
View file

@ -119,6 +119,8 @@ import com.raytheon.uf.edex.database.purge.PurgeLogger;
* the same parm simultaneously. * the same parm simultaneously.
* Added code to check the purge times when publishing and not publish * Added code to check the purge times when publishing and not publish
* data that is eligible to be purged. * data that is eligible to be purged.
* 05/22/14 #3071 randerso Expand publish time to time constraint quantum after truncating it
* to the purge time
* *
* </pre> * </pre>
* *
@ -172,7 +174,7 @@ public class GridParmManager {
this.lockMgr.setGridParmMgr(this); this.lockMgr.setGridParmMgr(this);
initializeManager(); initializeManager();
} }
/** /**
* Dispose the GridParmManager * Dispose the GridParmManager
@ -199,7 +201,7 @@ public class GridParmManager {
.debug("No matching GridDatabase for requested ParmID in createParm()"); .debug("No matching GridDatabase for requested ParmID in createParm()");
// TODO: should we return null? // TODO: should we return null?
return new GridParm(); return new GridParm();
} }
} }
/** /**
@ -330,10 +332,10 @@ public class GridParmManager {
for (SaveGridRequest req : saveRequest) { for (SaveGridRequest req : saveRequest) {
ServerResponse<?> ssr = null; ServerResponse<?> ssr = null;
GridParm gp = null; GridParm gp = null;
gp = gridParm(req.getParmId()); gp = gridParm(req.getParmId());
if (!gp.isValid()) { if (!gp.isValid()) {
sr.addMessage("Unknown Parm: " + req.getParmId() sr.addMessage("Unknown Parm: " + req.getParmId()
+ " in saveGridData()"); + " in saveGridData()");
statusHandler.error("Unknown Parm: " + req.getParmId() statusHandler.error("Unknown Parm: " + req.getParmId()
+ " in saveGridData()"); + " in saveGridData()");
continue; continue;
@ -455,27 +457,27 @@ public class GridParmManager {
// for the source data // for the source data
ParmID sourceParmId = req.getParmId(); ParmID sourceParmId = req.getParmId();
GridParm sourceGP = gridParm(sourceParmId); GridParm sourceGP = gridParm(sourceParmId);
if (!sourceGP.isValid()) { if (!sourceGP.isValid()) {
ssr.addMessage("Unknown Source Parm: " + req.getParmId() ssr.addMessage("Unknown Source Parm: " + req.getParmId()
+ " in commitGrid()"); + " in commitGrid()");
srDetailed.addMessages(ssr); srDetailed.addMessages(ssr);
failures.add(req); failures.add(req);
continue; continue;
} }
// for the destination data // for the destination data
ParmID destParmId = new ParmID(req.getParmId().getParmName(), ParmID destParmId = new ParmID(req.getParmId().getParmName(),
officialDBid, req.getParmId().getParmLevel()); officialDBid, req.getParmId().getParmLevel());
String destParmIdStr = destParmId.toString(); String destParmIdStr = destParmId.toString();
GridParm destGP = null; GridParm destGP = null;
destGP = gridParm(destParmId); destGP = gridParm(destParmId);
if (!destGP.isValid()) { if (!destGP.isValid()) {
ssr.addMessage("Unknown Destination Parm: " + destGP ssr.addMessage("Unknown Destination Parm: " + destGP
+ " in commitGrid()"); + " in commitGrid()");
srDetailed.addMessages(ssr); srDetailed.addMessages(ssr);
failures.add(req); failures.add(req);
continue; continue;
} }
// verify that the source and destination are matched // verify that the source and destination are matched
GridParmInfo sourceInfo, destInfo; GridParmInfo sourceInfo, destInfo;
@ -519,17 +521,22 @@ public class GridParmManager {
publishTime.setStart(startTime); publishTime.setStart(startTime);
} }
inventoryTimer.start(); // expand publishTime to TimeContraints so full grids are
// published
publishTime = sourceInfo.getTimeConstraints()
.expandTRToQuantum(publishTime);
inventoryTimer.start();
ServerResponse<List<TimeRange>> invSr = sourceGP ServerResponse<List<TimeRange>> invSr = sourceGP
.getGridInventory(publishTime); .getGridInventory(publishTime);
List<TimeRange> overlapInventory = invSr.getPayload(); List<TimeRange> overlapInventory = invSr.getPayload();
ssr.addMessages(invSr); ssr.addMessages(invSr);
if (!ssr.isOkay()) { if (!ssr.isOkay()) {
ssr.addMessage("GetGridInventory for source for commitGrid() failure: " ssr.addMessage("GetGridInventory for source for commitGrid() failure: "
+ ssr.message()); + ssr.message());
srDetailed.addMessages(ssr); srDetailed.addMessages(ssr);
failures.add(req); failures.add(req);
} }
// expand publish time to span overlapping inventory // expand publish time to span overlapping inventory
if (!overlapInventory.isEmpty()) { if (!overlapInventory.isEmpty()) {
@ -546,173 +553,173 @@ public class GridParmManager {
} }
invSr = destGP.getGridInventory(publishTime); invSr = destGP.getGridInventory(publishTime);
inventoryTimer.stop(); inventoryTimer.stop();
List<TimeRange> destInventory = invSr.getPayload(); List<TimeRange> destInventory = invSr.getPayload();
ssr.addMessages(invSr); ssr.addMessages(invSr);
if (!ssr.isOkay()) {
ssr.addMessage("GetGridInventory for destination for commitGrid() failure: "
+ ssr.message());
srDetailed.addMessages(ssr);
failures.add(req);
continue;
}
// get the source grid data
List<IGridSlice> sourceData = null;
List<TimeRange> badGridTR = new ArrayList<TimeRange>();
// System.out.println("overlapInventory initial size "
// + overlapInventory.size());
historyRetrieveTimer.start();
ServerResponse<Map<TimeRange, List<GridDataHistory>>> history = sourceGP
.getGridHistory(overlapInventory);
Map<TimeRange, List<GridDataHistory>> currentDestHistory = destGP
.getGridHistory(overlapInventory).getPayload();
historyRetrieveTimer.stop();
Map<TimeRange, List<GridDataHistory>> historyOnly = new HashMap<TimeRange, List<GridDataHistory>>();
for (TimeRange tr : history.getPayload().keySet()) {
// should only ever be one history for source grids
List<GridDataHistory> gdhList = history.getPayload()
.get(tr);
boolean doPublish = false;
for (GridDataHistory gdh : gdhList) {
// if update time is less than publish time, grid
// has not changed since last published,
// therefore only update history, do not publish
if ((gdh.getPublishTime() == null)
|| (gdh.getUpdateTime().getTime() > gdh
.getPublishTime().getTime())
// in service backup, times on srcHistory
// could appear as not needing a publish,
// even though dest data does not exist
|| (currentDestHistory.get(tr) == null)
|| (currentDestHistory.get(tr).size() == 0)) {
doPublish = true;
}
}
if (!doPublish) {
historyOnly.put(tr, gdhList);
overlapInventory.remove(tr);
}
}
retrieveTimer.start();
ServerResponse<List<IGridSlice>> getSr = sourceGP.getGridData(
new GetGridRequest(req.getParmId(), overlapInventory),
badGridTR);
retrieveTimer.stop();
// System.out.println("Retrieved " + overlapInventory.size()
// + " grids");
sourceData = getSr.getPayload();
ssr.addMessages(getSr);
if (!ssr.isOkay()) {
ssr.addMessage("GetGridData for source for commitGrid() failure: "
+ ssr.message());
srDetailed.addMessages(ssr);
failures.add(req);
continue;
}
// get list of official grids that overlap publish range and
// aren't contained in the publish range, these have to be
// included in the publish step. Then get the grids, shorten
// and insert into sourceData.
List<IGridSlice> officialData = new ArrayList<IGridSlice>();
List<TimeRange> officialTR = new ArrayList<TimeRange>();
for (int t = 0; t < destInventory.size(); t++) {
if (!publishTime.contains(destInventory.get(t))) {
officialTR.add(destInventory.get(t));
}
}
if (!officialTR.isEmpty()) {
retrieveTimer.start();
getSr = destGP.getGridData(new GetGridRequest(destParmId,
officialTR), badGridTR);
retrieveTimer.stop();
officialData = getSr.getPayload();
ssr.addMessages(getSr);
if (!ssr.isOkay()) { if (!ssr.isOkay()) {
ssr.addMessage("GetGridData for official for commidtGrid() failure: " ssr.addMessage("GetGridInventory for destination for commitGrid() failure: "
+ ssr.message()); + ssr.message());
srDetailed.addMessages(ssr); srDetailed.addMessages(ssr);
failures.add(req); failures.add(req);
continue; continue;
} }
// insert the grid into the "sourceGrid" list // get the source grid data
for (int t = 0; t < officialTR.size(); t++) { List<IGridSlice> sourceData = null;
// before List<TimeRange> badGridTR = new ArrayList<TimeRange>();
try {
if (officialTR.get(t).getStart() // System.out.println("overlapInventory initial size "
.before(publishTime.getStart())) { // + overlapInventory.size());
historyRetrieveTimer.start();
ServerResponse<Map<TimeRange, List<GridDataHistory>>> history = sourceGP
.getGridHistory(overlapInventory);
Map<TimeRange, List<GridDataHistory>> currentDestHistory = destGP
.getGridHistory(overlapInventory).getPayload();
historyRetrieveTimer.stop();
Map<TimeRange, List<GridDataHistory>> historyOnly = new HashMap<TimeRange, List<GridDataHistory>>();
for (TimeRange tr : history.getPayload().keySet()) {
// should only ever be one history for source grids
List<GridDataHistory> gdhList = history.getPayload()
.get(tr);
boolean doPublish = false;
for (GridDataHistory gdh : gdhList) {
// if update time is less than publish time, grid
// has not changed since last published,
// therefore only update history, do not publish
if ((gdh.getPublishTime() == null)
|| (gdh.getUpdateTime().getTime() > gdh
.getPublishTime().getTime())
// in service backup, times on srcHistory
// could appear as not needing a publish,
// even though dest data does not exist
|| (currentDestHistory.get(tr) == null)
|| (currentDestHistory.get(tr).size() == 0)) {
doPublish = true;
}
}
if (!doPublish) {
historyOnly.put(tr, gdhList);
overlapInventory.remove(tr);
}
}
retrieveTimer.start();
ServerResponse<List<IGridSlice>> getSr = sourceGP.getGridData(
new GetGridRequest(req.getParmId(), overlapInventory),
badGridTR);
retrieveTimer.stop();
// System.out.println("Retrieved " + overlapInventory.size()
// + " grids");
sourceData = getSr.getPayload();
ssr.addMessages(getSr);
if (!ssr.isOkay()) {
ssr.addMessage("GetGridData for source for commitGrid() failure: "
+ ssr.message());
srDetailed.addMessages(ssr);
failures.add(req);
continue;
}
// get list of official grids that overlap publish range and
// aren't contained in the publish range, these have to be
// included in the publish step. Then get the grids, shorten
// and insert into sourceData.
List<IGridSlice> officialData = new ArrayList<IGridSlice>();
List<TimeRange> officialTR = new ArrayList<TimeRange>();
for (int t = 0; t < destInventory.size(); t++) {
if (!publishTime.contains(destInventory.get(t))) {
officialTR.add(destInventory.get(t));
}
}
if (!officialTR.isEmpty()) {
retrieveTimer.start();
getSr = destGP.getGridData(new GetGridRequest(destParmId,
officialTR), badGridTR);
retrieveTimer.stop();
officialData = getSr.getPayload();
ssr.addMessages(getSr);
if (!ssr.isOkay()) {
ssr.addMessage("GetGridData for official for commidtGrid() failure: "
+ ssr.message());
srDetailed.addMessages(ssr);
failures.add(req);
continue;
}
// insert the grid into the "sourceGrid" list
for (int t = 0; t < officialTR.size(); t++) {
// before
try {
if (officialTR.get(t).getStart()
.before(publishTime.getStart())) {
IGridSlice tempSlice = officialData.get(t) IGridSlice tempSlice = officialData.get(t)
.clone(); .clone();
tempSlice.setValidTime(new TimeRange(officialTR tempSlice.setValidTime(new TimeRange(officialTR
.get(t).getStart(), publishTime .get(t).getStart(), publishTime
.getStart())); .getStart()));
sourceData.add(0, tempSlice); sourceData.add(0, tempSlice);
publishTime.setStart(officialTR.get(t) publishTime.setStart(officialTR.get(t)
.getStart()); .getStart());
overlapInventory.add(tempSlice.getValidTime()); overlapInventory.add(tempSlice.getValidTime());
} }
// after // after
if (officialTR.get(t).getEnd() if (officialTR.get(t).getEnd()
.after(publishTime.getEnd())) { .after(publishTime.getEnd())) {
IGridSlice tempSlice = officialData.get(t) IGridSlice tempSlice = officialData.get(t)
.clone(); .clone();
tempSlice.setValidTime(new TimeRange( tempSlice.setValidTime(new TimeRange(
publishTime.getEnd(), officialTR.get(t) publishTime.getEnd(), officialTR.get(t)
.getEnd())); .getEnd()));
sourceData.add(tempSlice); sourceData.add(tempSlice);
publishTime.setEnd(officialTR.get(t).getEnd()); publishTime.setEnd(officialTR.get(t).getEnd());
overlapInventory.add(tempSlice.getValidTime()); overlapInventory.add(tempSlice.getValidTime());
}
} catch (CloneNotSupportedException e) {
sr.addMessage("Error cloning GridSlice "
+ e.getMessage());
} }
} catch (CloneNotSupportedException e) {
sr.addMessage("Error cloning GridSlice "
+ e.getMessage());
} }
} }
}
// save off the source grid history, to update the source // save off the source grid history, to update the source
// database modify the source grid data for the dest ParmID and // database modify the source grid data for the dest ParmID and
// GridDataHistory // GridDataHistory
Map<TimeRange, List<GridDataHistory>> histories = new HashMap<TimeRange, List<GridDataHistory>>(); Map<TimeRange, List<GridDataHistory>> histories = new HashMap<TimeRange, List<GridDataHistory>>();
Date nowTime = new Date(); Date nowTime = new Date();
for (IGridSlice slice : sourceData) { for (IGridSlice slice : sourceData) {
GridDataHistory[] sliceHist = slice.getHistory(); GridDataHistory[] sliceHist = slice.getHistory();
for (GridDataHistory hist : sliceHist) { for (GridDataHistory hist : sliceHist) {
hist.setPublishTime((Date) nowTime.clone()); hist.setPublishTime((Date) nowTime.clone());
} }
slice.getGridInfo().resetParmID(destParmId); slice.getGridInfo().resetParmID(destParmId);
histories.put(slice.getValidTime(), histories.put(slice.getValidTime(),
Arrays.asList(sliceHist)); Arrays.asList(sliceHist));
} }
// update the history for publish time for grids that are // update the history for publish time for grids that are
// unchanged // unchanged
for (TimeRange tr : historyOnly.keySet()) { for (TimeRange tr : historyOnly.keySet()) {
List<GridDataHistory> histList = historyOnly.get(tr); List<GridDataHistory> histList = historyOnly.get(tr);
for (GridDataHistory hist : histList) { for (GridDataHistory hist : histList) {
hist.setPublishTime((Date) nowTime.clone()); hist.setPublishTime((Date) nowTime.clone());
}
histories.put(tr, histList);
} }
histories.put(tr, histList);
}
// update the publish times in the source database, // update the publish times in the source database,
// update the notifications // update the notifications
historyUpdateTimer.start(); historyUpdateTimer.start();
sr.addMessages(sourceGP.updatePublishTime(histories.values(), sr.addMessages(sourceGP.updatePublishTime(histories.values(),
(Date) nowTime.clone())); (Date) nowTime.clone()));
// System.out.println("Updated " + histories.size() + // System.out.println("Updated " + histories.size() +
// " histories"); // " histories");
historyUpdateTimer.stop(); historyUpdateTimer.stop();
List<TimeRange> historyTimes = new ArrayList<TimeRange>( List<TimeRange> historyTimes = new ArrayList<TimeRange>(
histories.keySet()); histories.keySet());
@ -723,56 +730,56 @@ public class GridParmManager {
// update the histories of destination database for ones // update the histories of destination database for ones
// that are not going to be saved since there hasn't been a // that are not going to be saved since there hasn't been a
// change // change
List<TimeRange> historyOnlyList = new ArrayList<TimeRange>(); List<TimeRange> historyOnlyList = new ArrayList<TimeRange>();
historyOnlyList.addAll(historyOnly.keySet()); historyOnlyList.addAll(historyOnly.keySet());
historyRetrieveTimer.start(); historyRetrieveTimer.start();
Map<TimeRange, List<GridDataHistory>> destHistory = destGP Map<TimeRange, List<GridDataHistory>> destHistory = destGP
.getGridHistory(historyOnlyList).getPayload(); .getGridHistory(historyOnlyList).getPayload();
historyRetrieveTimer.stop(); historyRetrieveTimer.stop();
for (TimeRange tr : destHistory.keySet()) { for (TimeRange tr : destHistory.keySet()) {
List<GridDataHistory> srcHistList = histories.get(tr); List<GridDataHistory> srcHistList = histories.get(tr);
List<GridDataHistory> destHistList = destHistory.get(tr); List<GridDataHistory> destHistList = destHistory.get(tr);
for (int i = 0; i < srcHistList.size(); i++) { for (int i = 0; i < srcHistList.size(); i++) {
destHistList.get(i).replaceValues(srcHistList.get(i)); destHistList.get(i).replaceValues(srcHistList.get(i));
}
} }
}
// only need to update the publish time on the destination // only need to update the publish time on the destination
// histories of grids that are not being saved (due to no // histories of grids that are not being saved (due to no
// changes), because the saveGridSlices() call below will update // changes), because the saveGridSlices() call below will update
// the publish time of the ones with changes // the publish time of the ones with changes
historyUpdateTimer.start(); historyUpdateTimer.start();
destGP.updatePublishTime(destHistory.values(), destGP.updatePublishTime(destHistory.values(),
(Date) nowTime.clone()); (Date) nowTime.clone());
historyUpdateTimer.stop(); historyUpdateTimer.stop();
// save data directly to the official database (bypassing // save data directly to the official database (bypassing
// the checks in Parm intentionally) // the checks in Parm intentionally)
storeTimer.start(); storeTimer.start();
ssr.addMessages(officialDBPtr.saveGridSlices(destParmId, ssr.addMessages(officialDBPtr.saveGridSlices(destParmId,
publishTime, sourceData, requestorId, historyOnlyList)); publishTime, sourceData, requestorId, historyOnlyList));
storeTimer.stop(); storeTimer.stop();
// System.out.println("Published " + sourceData.size() + // System.out.println("Published " + sourceData.size() +
// " slices"); // " slices");
if (!ssr.isOkay()) { if (!ssr.isOkay()) {
ssr.addMessage("SaveGridData for official for commitGrid() failure: " ssr.addMessage("SaveGridData for official for commitGrid() failure: "
+ ssr.message()); + ssr.message());
srDetailed.addMessages(ssr); srDetailed.addMessages(ssr);
failures.add(req); failures.add(req);
continue; continue;
} }
// make the notification // make the notification
GridUpdateNotification not = new GridUpdateNotification( GridUpdateNotification not = new GridUpdateNotification(
destParmId, publishTime, histories, requestorId, siteID); destParmId, publishTime, histories, requestorId, siteID);
changes.add(not); changes.add(not);
sr.getPayload().add(not); sr.getPayload().add(not);
} finally { } finally {
ClusterLockUtils.unlock(ct, false); ClusterLockUtils.unlock(ct, false);
} }
} }
perfLog.logDuration("Publish Grids: Acquiring cluster lock", perfLog.logDuration("Publish Grids: Acquiring cluster lock",
@ -822,8 +829,8 @@ public class GridParmManager {
this.dbMap.keySet()); this.dbMap.keySet());
sr.setPayload(databases); sr.setPayload(databases);
return sr; return sr;
} }
/** /**
* Get a database if available * Get a database if available
@ -847,8 +854,8 @@ public class GridParmManager {
if (status.isOkay()) { if (status.isOkay()) {
db = status.getPayload(); db = status.getPayload();
createDbNotification(Arrays.asList(dbId), null); createDbNotification(Arrays.asList(dbId), null);
}
} }
}
if (db != null) { if (db != null) {
this.addDB(db); this.addDB(db);
@ -886,8 +893,8 @@ public class GridParmManager {
return sr; return sr;
} }
return sr; return sr;
} }
/** /**
* Delete database * Delete database
@ -944,9 +951,9 @@ public class GridParmManager {
if (db == null) { if (db == null) {
sr.addMessage("Database " + dbId sr.addMessage("Database " + dbId
+ " does not exist for getParmList()"); + " does not exist for getParmList()");
return sr; return sr;
} }
sr = db.getParmList(); sr = db.getParmList();
return sr; return sr;
@ -988,7 +995,7 @@ public class GridParmManager {
// determine desired number of versions // determine desired number of versions
desiredVersions = this.config.desiredDbVersions(dbId); desiredVersions = this.config.desiredDbVersions(dbId);
} }
// process the id and determine whether it should be purged // process the id and determine whether it should be purged
count++; count++;
@ -1012,9 +1019,9 @@ public class GridParmManager {
toRemove.removeAll(newInv); toRemove.removeAll(newInv);
for (DatabaseID dbId : toRemove) { for (DatabaseID dbId : toRemove) {
if (dbMap.remove(dbId) != null) { if (dbMap.remove(dbId) != null) {
statusHandler statusHandler
.info("Synching GridParmManager with database inventory, removing " .info("Synching GridParmManager with database inventory, removing "
+ dbId); + dbId);
} }
// add any removals to the deletions list // add any removals to the deletions list
@ -1073,14 +1080,14 @@ public class GridParmManager {
List<LockNotification> lockNotify = new ArrayList<LockNotification>(); List<LockNotification> lockNotify = new ArrayList<LockNotification>();
GridParm gp = createParm(parmId); GridParm gp = createParm(parmId);
if (gp.isValid()) { if (gp.isValid()) {
ServerResponse<Integer> sr1 = gp.timePurge(purgeTime, ServerResponse<Integer> sr1 = gp.timePurge(purgeTime,
gridNotify, lockNotify); gridNotify, lockNotify);
sr.addMessages(sr1); sr.addMessages(sr1);
purgedCount += sr1.getPayload(); purgedCount += sr1.getPayload();
gridNotifications.addAll(gridNotify); gridNotifications.addAll(gridNotify);
lockNotifications.addAll(lockNotify); lockNotifications.addAll(lockNotify);
} }
} }
PurgeLogger.logInfo("Purge " + purgedCount + " items from " + dbId, PurgeLogger.logInfo("Purge " + purgedCount + " items from " + dbId,
@ -1119,7 +1126,7 @@ public class GridParmManager {
if (dbId.getRemovedDate() != null) { if (dbId.getRemovedDate() != null) {
// mark database as not removed // mark database as not removed
try { try {
GFEDao gfeDao = new GFEDao(); GFEDao gfeDao = new GFEDao();
gfeDao.setDatabaseRemovedDate(dbId, null); gfeDao.setDatabaseRemovedDate(dbId, null);
statusHandler.info("Database " + dbId + " restored"); statusHandler.info("Database " + dbId + " restored");
@ -1127,7 +1134,7 @@ public class GridParmManager {
statusHandler.handle(Priority.PROBLEM, statusHandler.handle(Priority.PROBLEM,
"Unable to mark database restored: " + dbId, e); "Unable to mark database restored: " + dbId, e);
} }
} }
// add to list of databases // add to list of databases
addDB(db); addDB(db);
@ -1177,8 +1184,8 @@ public class GridParmManager {
if (manID.getFormat().equals(DataType.GRID) if (manID.getFormat().equals(DataType.GRID)
&& !inventory.contains(manID)) { && !inventory.contains(manID)) {
inventory.add(manID); inventory.add(manID);
}
} }
}
// create the databases (the list should now only contain GRID dbs) // create the databases (the list should now only contain GRID dbs)
ServerResponse<GridDatabase> sr = new ServerResponse<GridDatabase>(); ServerResponse<GridDatabase> sr = new ServerResponse<GridDatabase>();
@ -1285,7 +1292,7 @@ public class GridParmManager {
if (db == null) { if (db == null) {
// New database // New database
db = D2DGridDatabase.getDatabase(config, d2dModelName, refTime); db = D2DGridDatabase.getDatabase(config, d2dModelName, refTime);
if (db == null) { if (db == null) {
continue; continue;
} }
@ -1308,16 +1315,16 @@ public class GridParmManager {
queue.queue(siteID, config, dbId, validTime, false, queue.queue(siteID, config, dbId, validTime, false,
SmartInitRecord.LIVE_SMART_INIT_PRIORITY); SmartInitRecord.LIVE_SMART_INIT_PRIORITY);
} }
} }
} }
// send notifications; // send notifications;
try { try {
SendNotifications.send(guns); SendNotifications.send(guns);
} catch (Exception e) { } catch (Exception e) {
statusHandler.error("Unable to send grib ingest notifications", e); statusHandler.error("Unable to send grib ingest notifications", e);
} }
} }
/** /**
* @param records * @param records
@ -1339,9 +1346,9 @@ public class GridParmManager {
Date validTime = gun.getReplacementTimeRange().getStart(); Date validTime = gun.getReplacementTimeRange().getStart();
queue.queue(siteID, config, dbId, validTime, false, queue.queue(siteID, config, dbId, validTime, false,
SmartInitRecord.LIVE_SMART_INIT_PRIORITY); SmartInitRecord.LIVE_SMART_INIT_PRIORITY);
}
} }
} }
}
try { try {
SendNotifications.send(guns); SendNotifications.send(guns);
@ -1349,7 +1356,7 @@ public class GridParmManager {
statusHandler.error( statusHandler.error(
"Unable to send satellite ingest notifications", e); "Unable to send satellite ingest notifications", e);
} }
} }
private Date purgeTime(DatabaseID id) { private Date purgeTime(DatabaseID id) {
int numHours = this.config.gridPurgeAgeInHours(id); int numHours = this.config.gridPurgeAgeInHours(id);
@ -1427,8 +1434,8 @@ public class GridParmManager {
for (ParmID pid : parmList) { for (ParmID pid : parmList) {
out.add(new CommitGridRequest(pid, req.getTimeRange(), out.add(new CommitGridRequest(pid, req.getTimeRange(),
req.isClientSendStatus())); req.isClientSendStatus()));
} }
} else { } else {
sr.addMessage("Could not find database for " sr.addMessage("Could not find database for "
+ req.getDbId() + " in convertToParmReq()"); + req.getDbId() + " in convertToParmReq()");
} }
@ -1544,7 +1551,7 @@ public class GridParmManager {
DatabaseID dbId = db.getDbId(); DatabaseID dbId = db.getDbId();
statusHandler.info("addDB called, adding " + dbId); statusHandler.info("addDB called, adding " + dbId);
this.dbMap.put(dbId, db); this.dbMap.put(dbId, db);
} }
/** /**
* Process D2D grid data purge notification * Process D2D grid data purge notification
@ -1568,9 +1575,9 @@ public class GridParmManager {
newInventory.addAll(dbIds); newInventory.addAll(dbIds);
} catch (DataAccessLayerException e) { } catch (DataAccessLayerException e) {
statusHandler.error(e.getLocalizedMessage(), e); statusHandler.error(e.getLocalizedMessage(), e);
}
} }
} }
}
DatabaseID satDbid = D2DSatDatabase.getDbId(siteID); DatabaseID satDbid = D2DSatDatabase.getDbId(siteID);
@ -1613,8 +1620,8 @@ public class GridParmManager {
statusHandler.info("d2dGridDataPurged removing database: " statusHandler.info("d2dGridDataPurged removing database: "
+ dbid); + dbid);
} }
}
} }
}
// if ((added.size() > 0) || (deleted.size() > 0)) { // if ((added.size() > 0) || (deleted.size() > 0)) {
// DBInvChangeNotification changed = new DBInvChangeNotification( // DBInvChangeNotification changed = new DBInvChangeNotification(
@ -1624,8 +1631,8 @@ public class GridParmManager {
deleted, siteID); deleted, siteID);
SendNotifications.send(changed); SendNotifications.send(changed);
}
} }
}
/** /**
* Process D2D satellite data purge notification * Process D2D satellite data purge notification

18
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/D2DGridDatabase.java
View file

@ -114,6 +114,7 @@ import com.raytheon.uf.edex.database.DataAccessLayerException;
* Added function to create a D2DGridDatabase object only if there is * Added function to create a D2DGridDatabase object only if there is
* data in postgres for the desired model/reftime * data in postgres for the desired model/reftime
* 04/17/2014 #2934 dgilling Change getGridParmInfo to use D2DParm's GridParmInfo. * 04/17/2014 #2934 dgilling Change getGridParmInfo to use D2DParm's GridParmInfo.
* 05/22/2014 #3071 randerso Improved error logging
* *
* </pre> * </pre>
* *
@ -808,10 +809,10 @@ public class D2DGridDatabase extends VGridDatabase {
long t0 = System.currentTimeMillis(); long t0 = System.currentTimeMillis();
Integer fcstHr = null;
try { try {
// Gets the metadata from the grib metadata database // Gets the metadata from the grib metadata database
D2DParm parm = this.gfeParms.get(parmId); D2DParm parm = this.gfeParms.get(parmId);
Integer fcstHr = null;
if (!GridPathProvider.STATIC_PARAMETERS.contains(parmId if (!GridPathProvider.STATIC_PARAMETERS.contains(parmId
.getParmName())) { .getParmName())) {
fcstHr = parm.getTimeRangeToFcstHr().get(timeRange); fcstHr = parm.getTimeRangeToFcstHr().get(timeRange);
@ -822,9 +823,10 @@ public class D2DGridDatabase extends VGridDatabase {
} }
d2dRecord = d2dDao.getGrid(d2dModelName, refTime, d2dRecord = d2dDao.getGrid(d2dModelName, refTime,
parm.getComponents()[0], parm.getLevel(), fcstHr, gpi); parm.getComponents()[0], parm.getLevel(), fcstHr, gpi);
} catch (DataAccessLayerException e) { } catch (Exception e) {
throw new GfeException( throw new GfeException(
"Error retrieving D2D Grid record from database", e); "Error retrieving D2D Grid record from database for "
+ parmId + " fcstHr: " + fcstHr, e);
} }
long t1 = System.currentTimeMillis(); long t1 = System.currentTimeMillis();
@ -964,9 +966,10 @@ public class D2DGridDatabase extends VGridDatabase {
throw new GfeException("Unable to remap UV wind grids", e); throw new GfeException("Unable to remap UV wind grids", e);
} }
return; return;
} catch (DataAccessLayerException e) { } catch (Exception e) {
throw new GfeException( throw new GfeException(
"Unable to retrieve wind grids from D2D database", e); "Unable to retrieve wind grids from D2D database for "
+ parmId + " fcstHr: " + fcstHr, e);
} }
} else { } else {
@ -999,9 +1002,10 @@ public class D2DGridDatabase extends VGridDatabase {
throw new GfeException("Unable to remap wind grids", e); throw new GfeException("Unable to remap wind grids", e);
} }
return; return;
} catch (DataAccessLayerException e) { } catch (Exception e) {
throw new GfeException( throw new GfeException(
"Unable to retrieve wind grids from D2D database", e); "Unable to retrieve wind grids from D2D database for "
+ parmId + " fcstHr: " + fcstHr, e);
} }
} }
} }

151
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/database/IFPGridDatabase.java
View file

@ -114,6 +114,7 @@ import com.raytheon.uf.edex.database.DataAccessLayerException;
* 08/05/13 #1571 randerso Refactored to store GridParmInfo and ParmStorageinfo in postgres database * 08/05/13 #1571 randerso Refactored to store GridParmInfo and ParmStorageinfo in postgres database
* 10/31/2013 #2508 randerso Change to use DiscreteGridSlice.getKeys() * 10/31/2013 #2508 randerso Change to use DiscreteGridSlice.getKeys()
* 12/10/13 #2611 randerso Change saveGridData to set update time when saving grids * 12/10/13 #2611 randerso Change saveGridData to set update time when saving grids
* 05/29/2014 #3071 randerso Fix NPE in getCachedParmID
* *
* </pre> * </pre>
* *
@ -154,22 +155,22 @@ public class IFPGridDatabase extends GridDatabase {
this.valid = true; this.valid = true;
ServerResponse<Object> failResponse = new ServerResponse<Object>(); ServerResponse<Object> failResponse = new ServerResponse<Object>();
try { try {
// lookup actual database id row from database // lookup actual database id row from database
// if it doesn't exist, it will be created at this point // if it doesn't exist, it will be created at this point
this.dao = new GFEDao(); this.dao = new GFEDao();
// Make a DatabaseID and save it. // Make a DatabaseID and save it.
this.dbId = dao.getDatabaseId(dbId); this.dbId = dao.getDatabaseId(dbId);
} catch (Exception e) { } catch (Exception e) {
String msg = "Unable to look up database id for ifp database: " String msg = "Unable to look up database id for ifp database: "
+ dbId; + dbId;
statusHandler.handle(Priority.PROBLEM, msg, e); statusHandler.handle(Priority.PROBLEM, msg, e);
failResponse.addMessage(msg); failResponse.addMessage(msg);
} }
if (!failInitCheck(failResponse)) { if (!failInitCheck(failResponse)) {
return; return;
} }
// Get the current database configuration and store the information // Get the current database configuration and store the information
// in private data _parmInfo, _parmStorageInfo, and _areaStorageInfo // in private data _parmInfo, _parmStorageInfo, and _areaStorageInfo
@ -220,7 +221,7 @@ public class IFPGridDatabase extends GridDatabase {
statusHandler.error("DatabaseFAIL: " + this.dbId + "\n" statusHandler.error("DatabaseFAIL: " + this.dbId + "\n"
+ failResponse.getMessages()); + failResponse.getMessages());
this.valid = false; this.valid = false;
} }
return this.valid; return this.valid;
} }
@ -574,19 +575,19 @@ public class IFPGridDatabase extends GridDatabase {
* The list of parms to delete * The list of parms to delete
*/ */
private void removeOldParms(List<String> parms) { private void removeOldParms(List<String> parms) {
for (String item : parms) { for (String item : parms) {
statusHandler.handle(Priority.INFO, "Removing: " + item statusHandler.handle(Priority.INFO, "Removing: " + item
+ " from the " + this.dbId + " database."); + " from the " + this.dbId + " database.");
try { try {
// Remove the entire data structure for the parm // Remove the entire data structure for the parm
dao.removeParm(parmStorageInfo.get(item).getParmID()); dao.removeParm(parmStorageInfo.get(item).getParmID());
this.parmStorageInfo.remove(item); this.parmStorageInfo.remove(item);
} catch (DataAccessLayerException e) { } catch (DataAccessLayerException e) {
statusHandler.handle(Priority.PROBLEM, "Error removing: " statusHandler.handle(Priority.PROBLEM, "Error removing: "
+ item + " from the database"); + item + " from the database");
}
} }
} }
}
@Override @Override
public ServerResponse<List<ParmID>> getParmList() { public ServerResponse<List<ParmID>> getParmList() {
@ -1138,7 +1139,7 @@ public class IFPGridDatabase extends GridDatabase {
if (!glocUser.equals(glocDb)) { if (!glocUser.equals(glocDb)) {
// save/update the database GridLocation // save/update the database GridLocation
try { try {
dao.saveOrUpdateGridLocation(glocUser); dao.saveOrUpdateGridLocation(glocUser);
// remap the actual gridded data to the new gridLocation // remap the actual gridded data to the new gridLocation
@ -1177,7 +1178,7 @@ public class IFPGridDatabase extends GridDatabase {
ParmStorageInfo newPSI = parmStorageInfoUser.get(compositeName); ParmStorageInfo newPSI = parmStorageInfoUser.get(compositeName);
if (newPSI == null) { if (newPSI == null) {
continue; // this parm not in new database, so skip continue; // this parm not in new database, so skip
} }
GridParmInfo newGPI = newPSI.getGridParmInfo(); GridParmInfo newGPI = newPSI.getGridParmInfo();
@ -1197,12 +1198,12 @@ public class IFPGridDatabase extends GridDatabase {
statusHandler.error("Unable to retrieve GFERecords for " statusHandler.error("Unable to retrieve GFERecords for "
+ compositeName, e); + compositeName, e);
continue; continue;
} }
// process each grid // process each grid
for (GFERecord rec : records) { for (GFERecord rec : records) {
List<TimeRange> times = new ArrayList<TimeRange>(); List<TimeRange> times = new ArrayList<TimeRange>();
times.add(rec.getTimeRange()); times.add(rec.getTimeRange());
ServerResponse<List<IGridSlice>> ssr = this.getGridData( ServerResponse<List<IGridSlice>> ssr = this.getGridData(
rec.getParmId(), times, oldGL); rec.getParmId(), times, oldGL);
sr.addMessages(ssr); sr.addMessages(ssr);
@ -1213,24 +1214,24 @@ public class IFPGridDatabase extends GridDatabase {
continue; continue;
} }
IGridSlice slice = ssr.getPayload().get(0); IGridSlice slice = ssr.getPayload().get(0);
IGridSlice newSlice = null; IGridSlice newSlice = null;
try { try {
switch (slice.getGridInfo().getGridType()) { switch (slice.getGridInfo().getGridType()) {
case NONE: case NONE:
break; break;
case SCALAR: case SCALAR:
ScalarGridSlice scalarSlice = (ScalarGridSlice) slice; ScalarGridSlice scalarSlice = (ScalarGridSlice) slice;
Grid2DFloat newGrid = remapper.remap(scalarSlice Grid2DFloat newGrid = remapper.remap(scalarSlice
.getScalarGrid(), scalarSlice.getGridInfo() .getScalarGrid(), scalarSlice.getGridInfo()
.getMinValue(), scalarSlice.getGridInfo() .getMinValue(), scalarSlice.getGridInfo()
.getMaxValue(), scalarSlice.getGridInfo() .getMaxValue(), scalarSlice.getGridInfo()
.getMinValue(), scalarSlice.getGridInfo() .getMinValue(), scalarSlice.getGridInfo()
.getMinValue()); .getMinValue());
scalarSlice.setScalarGrid(newGrid); scalarSlice.setScalarGrid(newGrid);
newSlice = scalarSlice; newSlice = scalarSlice;
break; break;
case VECTOR: case VECTOR:
VectorGridSlice vectorSlice = (VectorGridSlice) slice; VectorGridSlice vectorSlice = (VectorGridSlice) slice;
Grid2DFloat magOutput = new Grid2DFloat(newGL.getNx(), Grid2DFloat magOutput = new Grid2DFloat(newGL.getNx(),
newGL.getNy()); newGL.getNy());
Grid2DFloat dirOutput = new Grid2DFloat(newGL.getNx(), Grid2DFloat dirOutput = new Grid2DFloat(newGL.getNx(),
@ -1241,38 +1242,38 @@ public class IFPGridDatabase extends GridDatabase {
.getMaxValue(), vectorSlice.getGridInfo() .getMaxValue(), vectorSlice.getGridInfo()
.getMinValue(), vectorSlice.getGridInfo() .getMinValue(), vectorSlice.getGridInfo()
.getMinValue(), magOutput, dirOutput); .getMinValue(), magOutput, dirOutput);
vectorSlice.setDirGrid(dirOutput); vectorSlice.setDirGrid(dirOutput);
vectorSlice.setMagGrid(magOutput); vectorSlice.setMagGrid(magOutput);
newSlice = vectorSlice; newSlice = vectorSlice;
break; break;
case WEATHER: case WEATHER:
WeatherGridSlice weatherSlice = (WeatherGridSlice) slice; WeatherGridSlice weatherSlice = (WeatherGridSlice) slice;
Grid2DByte newWeatherGrid = remapper.remap( Grid2DByte newWeatherGrid = remapper.remap(
weatherSlice.getWeatherGrid(), 0, 0); weatherSlice.getWeatherGrid(), 0, 0);
weatherSlice.setWeatherGrid(newWeatherGrid); weatherSlice.setWeatherGrid(newWeatherGrid);
newSlice = weatherSlice; newSlice = weatherSlice;
break; break;
case DISCRETE: case DISCRETE:
DiscreteGridSlice discreteSlice = (DiscreteGridSlice) slice; DiscreteGridSlice discreteSlice = (DiscreteGridSlice) slice;
Grid2DByte newDiscreteGrid = remapper.remap( Grid2DByte newDiscreteGrid = remapper.remap(
discreteSlice.getDiscreteGrid(), 0, 0); discreteSlice.getDiscreteGrid(), 0, 0);
discreteSlice.setDiscreteGrid(newDiscreteGrid); discreteSlice.setDiscreteGrid(newDiscreteGrid);
newSlice = discreteSlice; newSlice = discreteSlice;
break; break;
}
newSlice.setGridInfo(newGPI);
rec.setMessageData(newSlice);
this.removeFromHDF5(rec);
this.saveGridsToHdf5(Arrays.asList(rec), newPSI);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Error remapping data for record [" + rec + "]", e);
}
} }
newSlice.setGridInfo(newGPI);
rec.setMessageData(newSlice);
this.removeFromHDF5(rec);
this.saveGridsToHdf5(Arrays.asList(rec), newPSI);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Error remapping data for record [" + rec + "]", e);
} }
}
}
return sr; return sr;
} }
private ServerResponse<?> getDBConfiguration() { private ServerResponse<?> getDBConfiguration() {
ServerResponse<?> sr = new ServerResponse<Object>(); ServerResponse<?> sr = new ServerResponse<Object>();
@ -1293,9 +1294,9 @@ public class IFPGridDatabase extends GridDatabase {
+ e.getLocalizedMessage(); + e.getLocalizedMessage();
statusHandler.error(msg, e); statusHandler.error(msg, e);
sr.addMessage(msg); sr.addMessage(msg);
}
return sr;
} }
return sr;
}
private void compareParmInfoWithDB( private void compareParmInfoWithDB(
Map<String, ParmStorageInfo> parmStorageInfoUser, Map<String, ParmStorageInfo> parmStorageInfoUser,
@ -1390,12 +1391,12 @@ public class IFPGridDatabase extends GridDatabase {
return null; return null;
} else { } else {
psi = this.gridDbConfig.getParmStorageInfo(nameLevel[0], psi = this.gridDbConfig.getParmStorageInfo(nameLevel[0],
nameLevel[1]); nameLevel[1]);
if (psi == null) { if (psi == null) {
statusHandler.handle(Priority.DEBUG, compositeName statusHandler.handle(Priority.DEBUG, compositeName
+ " not found in ParmStorageInfo config"); + " not found in ParmStorageInfo config");
return null; return null;
} }
} }
psi.getGridParmInfo().resetParmID( psi.getGridParmInfo().resetParmID(
@ -1726,7 +1727,7 @@ public class IFPGridDatabase extends GridDatabase {
first = false; first = false;
} else { } else {
sb.append(GfeUtil.KEY_SEPARATOR); sb.append(GfeUtil.KEY_SEPARATOR);
} }
sb.append(key.toString()); sb.append(key.toString());
} }
byte[] keyBytes = sb.toString().getBytes(); byte[] keyBytes = sb.toString().getBytes();
@ -2168,8 +2169,12 @@ public class IFPGridDatabase extends GridDatabase {
@Override @Override
public ParmID getCachedParmID(ParmID parmId) throws UnknownParmIdException { public ParmID getCachedParmID(ParmID parmId) throws UnknownParmIdException {
ParmID rval = this.parmStorageInfo.get(parmId.getCompositeName()) ParmID rval = null;
.getParmID(); ParmStorageInfo psi = this.parmStorageInfo.get(parmId
.getCompositeName());
if (psi != null) {
psi.getParmID();
}
if (rval == null) { if (rval == null) {
throw new UnknownParmIdException("ParmId: " + parmId.toString() throw new UnknownParmIdException("ParmId: " + parmId.toString()