Merge "Issue #1228 make PersistSrv better handle duplicates, make radar only allow overwrites on GSM products, make PyPies send back duplicate messages" into development
Former-commit-id:9a301f854f
[formerlyaa6ff7c959
[formerly 4b8f208e77c65deaeb2dc8128362af6a0624ccdb]] Former-commit-id:aa6ff7c959
Former-commit-id:4af47e6006
This commit is contained in:
commit
8df4a8ae5a
4 changed files with 282 additions and 243 deletions
|
@ -102,6 +102,7 @@ public class PersistSrv {
|
||||||
// All we know is something bad happened.
|
// All we know is something bad happened.
|
||||||
logger.error("Persistence error occurred: ", s);
|
logger.error("Persistence error occurred: ", s);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Produce error messages for each pdo that failed
|
// Produce error messages for each pdo that failed
|
||||||
int errCnt = 0;
|
int errCnt = 0;
|
||||||
|
@ -118,11 +119,11 @@ public class PersistSrv {
|
||||||
if (e.getValue() instanceof DuplicateRecordStorageException) {
|
if (e.getValue() instanceof DuplicateRecordStorageException) {
|
||||||
logger.warn("Duplicate record encountered (duplicate ignored): "
|
logger.warn("Duplicate record encountered (duplicate ignored): "
|
||||||
+ e.getKey().getDataURI());
|
+ e.getKey().getDataURI());
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
logger.error(
|
logger.error(
|
||||||
"Error persisting record " + e.getKey()
|
"Error persisting record " + e.getKey()
|
||||||
+ " to database: ",
|
+ " to database: ", e.getValue());
|
||||||
e.getValue());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -133,9 +134,6 @@ public class PersistSrv {
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (Throwable e1) {
|
} catch (Throwable e1) {
|
||||||
logger.error(
|
logger.error(
|
||||||
"Critical persistence error occurred. Individual records that failed will be logged separately.",
|
"Critical persistence error occurred. Individual records that failed will be logged separately.",
|
||||||
|
|
|
@ -510,7 +510,11 @@ public class RadarDecoder extends AbstractDecoder {
|
||||||
record.setPluginName("radar");
|
record.setPluginName("radar");
|
||||||
record.constructDataURI();
|
record.constructDataURI();
|
||||||
record.setInsertTime(TimeTools.getSystemCalendar());
|
record.setInsertTime(TimeTools.getSystemCalendar());
|
||||||
|
if (record.getProductCode() == 2) {
|
||||||
record.setOverwriteAllowed(true);
|
record.setOverwriteAllowed(true);
|
||||||
|
} else {
|
||||||
|
record.setOverwriteAllowed(false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -129,6 +129,7 @@ public class RadarDao extends PluginDao {
|
||||||
radarRec.getSymbologyBlock());
|
radarRec.getSymbologyBlock());
|
||||||
ByteDataRecord bdr = new ByteDataRecord(
|
ByteDataRecord bdr = new ByteDataRecord(
|
||||||
RadarStoredData.SYM_BLOCK_ID, radarRec.getDataURI(), data);
|
RadarStoredData.SYM_BLOCK_ID, radarRec.getDataURI(), data);
|
||||||
|
bdr.setCorrelationObject(radarRec);
|
||||||
dataStore.addDataRecord(bdr, sp);
|
dataStore.addDataRecord(bdr, sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -138,6 +139,7 @@ public class RadarDao extends PluginDao {
|
||||||
radarRec.getSymbologyData());
|
radarRec.getSymbologyData());
|
||||||
ByteDataRecord bdr = new ByteDataRecord(
|
ByteDataRecord bdr = new ByteDataRecord(
|
||||||
RadarStoredData.SYM_DATA_ID, radarRec.getDataURI(), data);
|
RadarStoredData.SYM_DATA_ID, radarRec.getDataURI(), data);
|
||||||
|
bdr.setCorrelationObject(radarRec);
|
||||||
dataStore.addDataRecord(bdr, sp);
|
dataStore.addDataRecord(bdr, sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -148,6 +150,7 @@ public class RadarDao extends PluginDao {
|
||||||
ByteDataRecord bdr = new ByteDataRecord(
|
ByteDataRecord bdr = new ByteDataRecord(
|
||||||
RadarStoredData.GRAPHIC_BLOCK_ID, radarRec.getDataURI(),
|
RadarStoredData.GRAPHIC_BLOCK_ID, radarRec.getDataURI(),
|
||||||
data);
|
data);
|
||||||
|
bdr.setCorrelationObject(radarRec);
|
||||||
dataStore.addDataRecord(bdr, sp);
|
dataStore.addDataRecord(bdr, sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -158,6 +161,7 @@ public class RadarDao extends PluginDao {
|
||||||
ByteDataRecord bdr = new ByteDataRecord(
|
ByteDataRecord bdr = new ByteDataRecord(
|
||||||
RadarStoredData.PRODUCT_VALS_ID, radarRec.getDataURI(),
|
RadarStoredData.PRODUCT_VALS_ID, radarRec.getDataURI(),
|
||||||
data);
|
data);
|
||||||
|
bdr.setCorrelationObject(radarRec);
|
||||||
dataStore.addDataRecord(bdr, sp);
|
dataStore.addDataRecord(bdr, sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -168,6 +172,7 @@ public class RadarDao extends PluginDao {
|
||||||
ByteDataRecord bdr = new ByteDataRecord(
|
ByteDataRecord bdr = new ByteDataRecord(
|
||||||
RadarStoredData.ALPHANUMERIC_ID, radarRec.getDataURI(),
|
RadarStoredData.ALPHANUMERIC_ID, radarRec.getDataURI(),
|
||||||
data);
|
data);
|
||||||
|
bdr.setCorrelationObject(radarRec);
|
||||||
dataStore.addDataRecord(bdr, sp);
|
dataStore.addDataRecord(bdr, sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -177,6 +182,7 @@ public class RadarDao extends PluginDao {
|
||||||
radarRec.getTabularBlock());
|
radarRec.getTabularBlock());
|
||||||
ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.TABULAR_ID,
|
ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.TABULAR_ID,
|
||||||
radarRec.getDataURI(), data);
|
radarRec.getDataURI(), data);
|
||||||
|
bdr.setCorrelationObject(radarRec);
|
||||||
dataStore.addDataRecord(bdr, sp);
|
dataStore.addDataRecord(bdr, sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -195,6 +201,7 @@ public class RadarDao extends PluginDao {
|
||||||
radarRec.getMapRecordVals());
|
radarRec.getMapRecordVals());
|
||||||
ByteDataRecord bdr = new ByteDataRecord(
|
ByteDataRecord bdr = new ByteDataRecord(
|
||||||
RadarStoredData.RECORD_VALS_ID, radarRec.getDataURI(), data);
|
RadarStoredData.RECORD_VALS_ID, radarRec.getDataURI(), data);
|
||||||
|
bdr.setCorrelationObject(radarRec);
|
||||||
dataStore.addDataRecord(bdr, sp);
|
dataStore.addDataRecord(bdr, sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -203,6 +210,7 @@ public class RadarDao extends PluginDao {
|
||||||
SerializationType.Thrift).serialize(radarRec.getStormIDs());
|
SerializationType.Thrift).serialize(radarRec.getStormIDs());
|
||||||
ByteDataRecord bdr = new ByteDataRecord(
|
ByteDataRecord bdr = new ByteDataRecord(
|
||||||
RadarStoredData.STORM_IDS_ID, radarRec.getDataURI(), data);
|
RadarStoredData.STORM_IDS_ID, radarRec.getDataURI(), data);
|
||||||
|
bdr.setCorrelationObject(radarRec);
|
||||||
dataStore.addDataRecord(bdr, sp);
|
dataStore.addDataRecord(bdr, sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -212,6 +220,7 @@ public class RadarDao extends PluginDao {
|
||||||
radarRec.getGsmMessage());
|
radarRec.getGsmMessage());
|
||||||
ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.GSM_ID,
|
ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.GSM_ID,
|
||||||
radarRec.getDataURI(), data);
|
radarRec.getDataURI(), data);
|
||||||
|
bdr.setCorrelationObject(radarRec);
|
||||||
dataStore.addDataRecord(bdr, sp);
|
dataStore.addDataRecord(bdr, sp);
|
||||||
}
|
}
|
||||||
if (radarRec.getAlertMessage() != null) {
|
if (radarRec.getAlertMessage() != null) {
|
||||||
|
@ -221,6 +230,7 @@ public class RadarDao extends PluginDao {
|
||||||
ByteDataRecord bdr = new ByteDataRecord(
|
ByteDataRecord bdr = new ByteDataRecord(
|
||||||
RadarStoredData.ALERT_MESSAGE_ID, radarRec.getDataURI(),
|
RadarStoredData.ALERT_MESSAGE_ID, radarRec.getDataURI(),
|
||||||
data);
|
data);
|
||||||
|
bdr.setCorrelationObject(radarRec);
|
||||||
dataStore.addDataRecord(bdr, sp);
|
dataStore.addDataRecord(bdr, sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -230,6 +240,7 @@ public class RadarDao extends PluginDao {
|
||||||
radarRec.getAapMessage());
|
radarRec.getAapMessage());
|
||||||
ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.AAP_ID,
|
ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.AAP_ID,
|
||||||
radarRec.getDataURI(), data);
|
radarRec.getDataURI(), data);
|
||||||
|
bdr.setCorrelationObject(radarRec);
|
||||||
dataStore.addDataRecord(bdr, sp);
|
dataStore.addDataRecord(bdr, sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,10 +3,12 @@ package com.raytheon.uf.common.pypies;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import com.raytheon.uf.common.comm.HttpClient;
|
import com.raytheon.uf.common.comm.HttpClient;
|
||||||
|
import com.raytheon.uf.common.datastorage.DuplicateRecordStorageException;
|
||||||
import com.raytheon.uf.common.datastorage.IDataStore;
|
import com.raytheon.uf.common.datastorage.IDataStore;
|
||||||
import com.raytheon.uf.common.datastorage.Request;
|
import com.raytheon.uf.common.datastorage.Request;
|
||||||
import com.raytheon.uf.common.datastorage.StorageException;
|
import com.raytheon.uf.common.datastorage.StorageException;
|
||||||
|
@ -277,14 +279,38 @@ public class PyPiesDataStore implements IDataStore {
|
||||||
StorageStatus ss = null;
|
StorageStatus ss = null;
|
||||||
try {
|
try {
|
||||||
StoreResponse sr = (StoreResponse) sendRequest(req);
|
StoreResponse sr = (StoreResponse) sendRequest(req);
|
||||||
records.clear();
|
|
||||||
ss = sr.getStatus();
|
ss = sr.getStatus();
|
||||||
String[] exc = sr.getExceptions();
|
String[] exc = sr.getExceptions();
|
||||||
IDataRecord[] failed = sr.getFailedRecords();
|
IDataRecord[] failed = sr.getFailedRecords();
|
||||||
|
|
||||||
|
// need to set the correlation object
|
||||||
|
if (failed != null) {
|
||||||
|
for (IDataRecord rec : failed) {
|
||||||
|
Iterator<IDataRecord> recordIter = records.iterator();
|
||||||
|
while (recordIter.hasNext()) {
|
||||||
|
IDataRecord oldRec = recordIter.next();
|
||||||
|
if (oldRec.getGroup().equals(rec.getGroup())
|
||||||
|
&& oldRec.getName().equals(rec.getName())) {
|
||||||
|
rec.setCorrelationObject(oldRec
|
||||||
|
.getCorrelationObject());
|
||||||
|
recordIter.remove();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
records.clear();
|
||||||
StorageException[] jexc = new StorageException[exc.length];
|
StorageException[] jexc = new StorageException[exc.length];
|
||||||
for (int i = 0; i < exc.length; i++) {
|
for (int i = 0; i < exc.length; i++) {
|
||||||
|
// checking for duplicates based on what is in the string...
|
||||||
|
if (exc[i].contains("already exists")) {
|
||||||
|
jexc[i] = new DuplicateRecordStorageException(exc[i],
|
||||||
|
failed[i]);
|
||||||
|
} else {
|
||||||
jexc[i] = new StorageException(exc[i], failed[i]);
|
jexc[i] = new StorageException(exc[i], failed[i]);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
ss.setExceptions(jexc);
|
ss.setExceptions(jexc);
|
||||||
} catch (StorageException e) {
|
} catch (StorageException e) {
|
||||||
|
|
Loading…
Add table
Reference in a new issue