Merge "Issue #1228 make PersistSrv better handle duplicates, make radar only allow overwrites on GSM products, make PyPies send back duplicate messages" into development

Former-commit-id: 8df4a8ae5a [formerly 4af47e6006] [formerly aa6ff7c959] [formerly 9a301f854f [formerly aa6ff7c959 [formerly 4b8f208e77c65deaeb2dc8128362af6a0624ccdb]]]
Former-commit-id: 9a301f854f
Former-commit-id: a641b3058e5cfe2c7e112daf8564e39f42a0e768 [formerly c6442f9009]
Former-commit-id: b9a69932fd

edexOsgi/com.raytheon.edex.ingestsrv/src/com/raytheon/edex/services/PersistSrv.java

@@ -102,6 +102,7 @@ public class PersistSrv {
                         // All we know is something bad happened.
                         logger.error("Persistence error occurred: ", s);
                     }
+                }
 
                 // Produce error messages for each pdo that failed
                 int errCnt = 0;
@@ -118,11 +119,11 @@ public class PersistSrv {
                         if (e.getValue() instanceof DuplicateRecordStorageException) {
                             logger.warn("Duplicate record encountered (duplicate ignored): "
                                     + e.getKey().getDataURI());
+
                         } else {
                             logger.error(
                                     "Error persisting record " + e.getKey()
-                                                + " to database: ",
+                                            + " to database: ", e.getValue());
-                                        e.getValue());
                         }
                     }
 
@@ -133,9 +134,6 @@ public class PersistSrv {
 
                 }
             }
-
-            }
-
         } catch (Throwable e1) {
             logger.error(
                     "Critical persistence error occurred.  Individual records that failed will be logged separately.",

edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/RadarDecoder.java

@@ -510,7 +510,11 @@ public class RadarDecoder extends AbstractDecoder {
         record.setPluginName("radar");
         record.constructDataURI();
         record.setInsertTime(TimeTools.getSystemCalendar());
+        if (record.getProductCode() == 2) {
             record.setOverwriteAllowed(true);
+        } else {
+            record.setOverwriteAllowed(false);
+        }
     }
 
     /**

edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/dao/RadarDao.java

@@ -129,6 +129,7 @@ public class RadarDao extends PluginDao {
                     radarRec.getSymbologyBlock());
             ByteDataRecord bdr = new ByteDataRecord(
                     RadarStoredData.SYM_BLOCK_ID, radarRec.getDataURI(), data);
+            bdr.setCorrelationObject(radarRec);
             dataStore.addDataRecord(bdr, sp);
         }
 
@@ -138,6 +139,7 @@ public class RadarDao extends PluginDao {
                     radarRec.getSymbologyData());
             ByteDataRecord bdr = new ByteDataRecord(
                     RadarStoredData.SYM_DATA_ID, radarRec.getDataURI(), data);
+            bdr.setCorrelationObject(radarRec);
             dataStore.addDataRecord(bdr, sp);
         }
 
@@ -148,6 +150,7 @@ public class RadarDao extends PluginDao {
             ByteDataRecord bdr = new ByteDataRecord(
                     RadarStoredData.GRAPHIC_BLOCK_ID, radarRec.getDataURI(),
                     data);
+            bdr.setCorrelationObject(radarRec);
             dataStore.addDataRecord(bdr, sp);
         }
 
@@ -158,6 +161,7 @@ public class RadarDao extends PluginDao {
             ByteDataRecord bdr = new ByteDataRecord(
                     RadarStoredData.PRODUCT_VALS_ID, radarRec.getDataURI(),
                     data);
+            bdr.setCorrelationObject(radarRec);
             dataStore.addDataRecord(bdr, sp);
         }
 
@@ -168,6 +172,7 @@ public class RadarDao extends PluginDao {
             ByteDataRecord bdr = new ByteDataRecord(
                     RadarStoredData.ALPHANUMERIC_ID, radarRec.getDataURI(),
                     data);
+            bdr.setCorrelationObject(radarRec);
             dataStore.addDataRecord(bdr, sp);
         }
 
@@ -177,6 +182,7 @@ public class RadarDao extends PluginDao {
                     radarRec.getTabularBlock());
             ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.TABULAR_ID,
                     radarRec.getDataURI(), data);
+            bdr.setCorrelationObject(radarRec);
             dataStore.addDataRecord(bdr, sp);
         }
 
@@ -195,6 +201,7 @@ public class RadarDao extends PluginDao {
                     radarRec.getMapRecordVals());
             ByteDataRecord bdr = new ByteDataRecord(
                     RadarStoredData.RECORD_VALS_ID, radarRec.getDataURI(), data);
+            bdr.setCorrelationObject(radarRec);
             dataStore.addDataRecord(bdr, sp);
         }
 
@@ -203,6 +210,7 @@ public class RadarDao extends PluginDao {
                     SerializationType.Thrift).serialize(radarRec.getStormIDs());
             ByteDataRecord bdr = new ByteDataRecord(
                     RadarStoredData.STORM_IDS_ID, radarRec.getDataURI(), data);
+            bdr.setCorrelationObject(radarRec);
             dataStore.addDataRecord(bdr, sp);
         }
 
@@ -212,6 +220,7 @@ public class RadarDao extends PluginDao {
                     radarRec.getGsmMessage());
             ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.GSM_ID,
                     radarRec.getDataURI(), data);
+            bdr.setCorrelationObject(radarRec);
             dataStore.addDataRecord(bdr, sp);
         }
         if (radarRec.getAlertMessage() != null) {
@@ -221,6 +230,7 @@ public class RadarDao extends PluginDao {
             ByteDataRecord bdr = new ByteDataRecord(
                     RadarStoredData.ALERT_MESSAGE_ID, radarRec.getDataURI(),
                     data);
+            bdr.setCorrelationObject(radarRec);
             dataStore.addDataRecord(bdr, sp);
         }
 
@@ -230,6 +240,7 @@ public class RadarDao extends PluginDao {
                     radarRec.getAapMessage());
             ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.AAP_ID,
                     radarRec.getDataURI(), data);
+            bdr.setCorrelationObject(radarRec);
             dataStore.addDataRecord(bdr, sp);
         }
 

edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/PyPiesDataStore.java

@@ -3,10 +3,12 @@ package com.raytheon.uf.common.pypies;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
 import com.raytheon.uf.common.comm.HttpClient;
+import com.raytheon.uf.common.datastorage.DuplicateRecordStorageException;
 import com.raytheon.uf.common.datastorage.IDataStore;
 import com.raytheon.uf.common.datastorage.Request;
 import com.raytheon.uf.common.datastorage.StorageException;
@@ -277,14 +279,38 @@ public class PyPiesDataStore implements IDataStore {
         StorageStatus ss = null;
         try {
             StoreResponse sr = (StoreResponse) sendRequest(req);
-            records.clear();
             ss = sr.getStatus();
             String[] exc = sr.getExceptions();
             IDataRecord[] failed = sr.getFailedRecords();
+
+            // need to set the correlation object
+            if (failed != null) {
+                for (IDataRecord rec : failed) {
+                    Iterator<IDataRecord> recordIter = records.iterator();
+                    while (recordIter.hasNext()) {
+                        IDataRecord oldRec = recordIter.next();
+                        if (oldRec.getGroup().equals(rec.getGroup())
+                                && oldRec.getName().equals(rec.getName())) {
+                            rec.setCorrelationObject(oldRec
+                                    .getCorrelationObject());
+                            recordIter.remove();
+                            break;
+                        }
+                    }
+                }
+            }
+
+            records.clear();
             StorageException[] jexc = new StorageException[exc.length];
             for (int i = 0; i < exc.length; i++) {
+                // checking for duplicates based on what is in the string...
+                if (exc[i].contains("already exists")) {
+                    jexc[i] = new DuplicateRecordStorageException(exc[i],
+                            failed[i]);
+                } else {
                     jexc[i] = new StorageException(exc[i], failed[i]);
                 }
+            }
 
             ss.setExceptions(jexc);
         } catch (StorageException e) {