diff --git a/edexOsgi/com.raytheon.edex.ingestsrv/src/com/raytheon/edex/services/PersistSrv.java b/edexOsgi/com.raytheon.edex.ingestsrv/src/com/raytheon/edex/services/PersistSrv.java index 2bcbebd228..48d4218703 100644 --- a/edexOsgi/com.raytheon.edex.ingestsrv/src/com/raytheon/edex/services/PersistSrv.java +++ b/edexOsgi/com.raytheon.edex.ingestsrv/src/com/raytheon/edex/services/PersistSrv.java @@ -102,40 +102,38 @@ public class PersistSrv { // All we know is something bad happened. logger.error("Persistence error occurred: ", s); } - - // Produce error messages for each pdo that failed - int errCnt = 0; - boolean suppressed = false; - for (Map.Entry e : pdosThatFailed - .entrySet()) { - if (errCnt > 50) { - logger.warn("More than 50 errors occurred in this batch. The remaining errors will be suppressed."); - suppressed = true; - continue; - } - - if (!suppressed) { - if (e.getValue() instanceof DuplicateRecordStorageException) { - logger.warn("Duplicate record encountered (duplicate ignored): " - + e.getKey().getDataURI()); - } else { - logger.error( - "Error persisting record " + e.getKey() - + " to database: ", - e.getValue()); - } - } - - // Remove from the pdoList so the pdo is not propagated - // to the next service - pdoList.remove(e.getKey()); - errCnt++; - - } } - } + // Produce error messages for each pdo that failed + int errCnt = 0; + boolean suppressed = false; + for (Map.Entry e : pdosThatFailed + .entrySet()) { + if (errCnt > 50) { + logger.warn("More than 50 errors occurred in this batch. The remaining errors will be suppressed."); + suppressed = true; + continue; + } + if (!suppressed) { + if (e.getValue() instanceof DuplicateRecordStorageException) { + logger.warn("Duplicate record encountered (duplicate ignored): " + + e.getKey().getDataURI()); + + } else { + logger.error( + "Error persisting record " + e.getKey() + + " to database: ", e.getValue()); + } + } + + // Remove from the pdoList so the pdo is not propagated + // to the next service + pdoList.remove(e.getKey()); + errCnt++; + + } + } } catch (Throwable e1) { logger.error( "Critical persistence error occurred. Individual records that failed will be logged separately.", diff --git a/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/RadarDecoder.java b/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/RadarDecoder.java index 047e27df30..55dcefe7b6 100644 --- a/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/RadarDecoder.java +++ b/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/RadarDecoder.java @@ -510,7 +510,11 @@ public class RadarDecoder extends AbstractDecoder { record.setPluginName("radar"); record.constructDataURI(); record.setInsertTime(TimeTools.getSystemCalendar()); - record.setOverwriteAllowed(true); + if (record.getProductCode() == 2) { + record.setOverwriteAllowed(true); + } else { + record.setOverwriteAllowed(false); + } } /** diff --git a/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/dao/RadarDao.java b/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/dao/RadarDao.java index 682fa4701b..3a0e30529d 100644 --- a/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/dao/RadarDao.java +++ b/edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/dao/RadarDao.java @@ -64,238 +64,249 @@ import com.raytheon.uf.edex.database.query.DatabaseQuery; public class RadarDao extends PluginDao { - /** - * Creates a new radar dao - * - * @param pluginName - * "radar" - * @throws PluginException - * If the dao cannot be initialized - */ - public RadarDao(String pluginName) throws PluginException { - super(pluginName); - } + /** + * Creates a new radar dao + * + * @param pluginName + * "radar" + * @throws PluginException + * If the dao cannot be initialized + */ + public RadarDao(String pluginName) throws PluginException { + super(pluginName); + } - @Override - protected IDataStore populateDataStore(IDataStore dataStore, - IPersistable obj) throws Exception { - RadarRecord radarRec = (RadarRecord) obj; - StorageProperties sp = null; - String compression = PluginRegistry.getInstance() - .getRegisteredObject(pluginName).getCompression(); - if (compression != null) { - sp = new StorageProperties(); - sp.setCompression(StorageProperties.Compression - .valueOf(compression)); - } - if (radarRec.getRawData() != null) { - IDataRecord rec = new ByteDataRecord(RadarStoredData.RAW_DATA_ID, - radarRec.getDataURI(), radarRec.getRawData(), 2, - new long[] { radarRec.getNumRadials(), - radarRec.getNumBins() }); - rec.setCorrelationObject(radarRec); - dataStore.addDataRecord(rec, sp); - } + @Override + protected IDataStore populateDataStore(IDataStore dataStore, + IPersistable obj) throws Exception { + RadarRecord radarRec = (RadarRecord) obj; + StorageProperties sp = null; + String compression = PluginRegistry.getInstance() + .getRegisteredObject(pluginName).getCompression(); + if (compression != null) { + sp = new StorageProperties(); + sp.setCompression(StorageProperties.Compression + .valueOf(compression)); + } + if (radarRec.getRawData() != null) { + IDataRecord rec = new ByteDataRecord(RadarStoredData.RAW_DATA_ID, + radarRec.getDataURI(), radarRec.getRawData(), 2, + new long[] { radarRec.getNumRadials(), + radarRec.getNumBins() }); + rec.setCorrelationObject(radarRec); + dataStore.addDataRecord(rec, sp); + } - if (radarRec.getRawShortData() != null) { - IDataRecord rec = new ShortDataRecord( - RadarStoredData.SHORT_DATA_ID, radarRec.getDataURI(), - radarRec.getRawShortData(), 2, new long[] { - radarRec.getNumRadials(), radarRec.getNumBins() }); - rec.setCorrelationObject(radarRec); - dataStore.addDataRecord(rec, sp); - } + if (radarRec.getRawShortData() != null) { + IDataRecord rec = new ShortDataRecord( + RadarStoredData.SHORT_DATA_ID, radarRec.getDataURI(), + radarRec.getRawShortData(), 2, new long[] { + radarRec.getNumRadials(), radarRec.getNumBins() }); + rec.setCorrelationObject(radarRec); + dataStore.addDataRecord(rec, sp); + } - if (radarRec.getAngleData() != null) { - IDataRecord rec = new FloatDataRecord( - RadarStoredData.ANGLE_DATA_ID, radarRec.getDataURI(), - radarRec.getAngleData(), 1, - new long[] { radarRec.getNumRadials() }); - rec.setCorrelationObject(radarRec); - dataStore.addDataRecord(rec, sp); - } + if (radarRec.getAngleData() != null) { + IDataRecord rec = new FloatDataRecord( + RadarStoredData.ANGLE_DATA_ID, radarRec.getDataURI(), + radarRec.getAngleData(), 1, + new long[] { radarRec.getNumRadials() }); + rec.setCorrelationObject(radarRec); + dataStore.addDataRecord(rec, sp); + } - if (radarRec.getThresholds() != null && radarRec.getProductCode() != 2) { - IDataRecord rec = new ShortDataRecord( - RadarStoredData.THRESHOLDS_ID, radarRec.getDataURI(), - radarRec.getThresholds(), 1, new long[] { 16 }); - rec.setCorrelationObject(radarRec); - dataStore.addDataRecord(rec, sp); - } + if (radarRec.getThresholds() != null && radarRec.getProductCode() != 2) { + IDataRecord rec = new ShortDataRecord( + RadarStoredData.THRESHOLDS_ID, radarRec.getDataURI(), + radarRec.getThresholds(), 1, new long[] { 16 }); + rec.setCorrelationObject(radarRec); + dataStore.addDataRecord(rec, sp); + } - if (radarRec.getSymbologyBlock() != null) { - byte[] data = DynamicSerializationManager.getManager( - SerializationType.Thrift).serialize( - radarRec.getSymbologyBlock()); - ByteDataRecord bdr = new ByteDataRecord( - RadarStoredData.SYM_BLOCK_ID, radarRec.getDataURI(), data); - dataStore.addDataRecord(bdr, sp); - } + if (radarRec.getSymbologyBlock() != null) { + byte[] data = DynamicSerializationManager.getManager( + SerializationType.Thrift).serialize( + radarRec.getSymbologyBlock()); + ByteDataRecord bdr = new ByteDataRecord( + RadarStoredData.SYM_BLOCK_ID, radarRec.getDataURI(), data); + bdr.setCorrelationObject(radarRec); + dataStore.addDataRecord(bdr, sp); + } - if (radarRec.getSymbologyData() != null) { - byte[] data = DynamicSerializationManager.getManager( - SerializationType.Thrift).serialize( - radarRec.getSymbologyData()); - ByteDataRecord bdr = new ByteDataRecord( - RadarStoredData.SYM_DATA_ID, radarRec.getDataURI(), data); - dataStore.addDataRecord(bdr, sp); - } + if (radarRec.getSymbologyData() != null) { + byte[] data = DynamicSerializationManager.getManager( + SerializationType.Thrift).serialize( + radarRec.getSymbologyData()); + ByteDataRecord bdr = new ByteDataRecord( + RadarStoredData.SYM_DATA_ID, radarRec.getDataURI(), data); + bdr.setCorrelationObject(radarRec); + dataStore.addDataRecord(bdr, sp); + } - if (radarRec.getGraphicBlock() != null) { - byte[] data = DynamicSerializationManager.getManager( - SerializationType.Thrift).serialize( - radarRec.getGraphicBlock()); - ByteDataRecord bdr = new ByteDataRecord( - RadarStoredData.GRAPHIC_BLOCK_ID, radarRec.getDataURI(), - data); - dataStore.addDataRecord(bdr, sp); - } + if (radarRec.getGraphicBlock() != null) { + byte[] data = DynamicSerializationManager.getManager( + SerializationType.Thrift).serialize( + radarRec.getGraphicBlock()); + ByteDataRecord bdr = new ByteDataRecord( + RadarStoredData.GRAPHIC_BLOCK_ID, radarRec.getDataURI(), + data); + bdr.setCorrelationObject(radarRec); + dataStore.addDataRecord(bdr, sp); + } - if (radarRec.getMapProductVals() != null) { - byte[] data = DynamicSerializationManager.getManager( - SerializationType.Thrift).serialize( - radarRec.getMapProductVals()); - ByteDataRecord bdr = new ByteDataRecord( - RadarStoredData.PRODUCT_VALS_ID, radarRec.getDataURI(), - data); - dataStore.addDataRecord(bdr, sp); - } + if (radarRec.getMapProductVals() != null) { + byte[] data = DynamicSerializationManager.getManager( + SerializationType.Thrift).serialize( + radarRec.getMapProductVals()); + ByteDataRecord bdr = new ByteDataRecord( + RadarStoredData.PRODUCT_VALS_ID, radarRec.getDataURI(), + data); + bdr.setCorrelationObject(radarRec); + dataStore.addDataRecord(bdr, sp); + } - if (radarRec.getAlphanumericValues() != null) { - byte[] data = DynamicSerializationManager.getManager( - SerializationType.Thrift).serialize( - radarRec.getAlphanumericValues()); - ByteDataRecord bdr = new ByteDataRecord( - RadarStoredData.ALPHANUMERIC_ID, radarRec.getDataURI(), - data); - dataStore.addDataRecord(bdr, sp); - } + if (radarRec.getAlphanumericValues() != null) { + byte[] data = DynamicSerializationManager.getManager( + SerializationType.Thrift).serialize( + radarRec.getAlphanumericValues()); + ByteDataRecord bdr = new ByteDataRecord( + RadarStoredData.ALPHANUMERIC_ID, radarRec.getDataURI(), + data); + bdr.setCorrelationObject(radarRec); + dataStore.addDataRecord(bdr, sp); + } - if (radarRec.getTabularBlock() != null) { - byte[] data = DynamicSerializationManager.getManager( - SerializationType.Thrift).serialize( - radarRec.getTabularBlock()); - ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.TABULAR_ID, - radarRec.getDataURI(), data); - dataStore.addDataRecord(bdr, sp); - } + if (radarRec.getTabularBlock() != null) { + byte[] data = DynamicSerializationManager.getManager( + SerializationType.Thrift).serialize( + radarRec.getTabularBlock()); + ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.TABULAR_ID, + radarRec.getDataURI(), data); + bdr.setCorrelationObject(radarRec); + dataStore.addDataRecord(bdr, sp); + } - if (radarRec.getProductDependentValues() != null) { - IDataRecord rec = new ShortDataRecord( - RadarStoredData.DEPENDENT_VALS_ID, radarRec.getDataURI(), - radarRec.getProductDependentValues(), 1, - new long[] { radarRec.getProductDependentValues().length }); - rec.setCorrelationObject(radarRec); - dataStore.addDataRecord(rec, sp); - } + if (radarRec.getProductDependentValues() != null) { + IDataRecord rec = new ShortDataRecord( + RadarStoredData.DEPENDENT_VALS_ID, radarRec.getDataURI(), + radarRec.getProductDependentValues(), 1, + new long[] { radarRec.getProductDependentValues().length }); + rec.setCorrelationObject(radarRec); + dataStore.addDataRecord(rec, sp); + } - if (radarRec.getMapRecordVals() != null) { - byte[] data = DynamicSerializationManager.getManager( - SerializationType.Thrift).serialize( - radarRec.getMapRecordVals()); - ByteDataRecord bdr = new ByteDataRecord( - RadarStoredData.RECORD_VALS_ID, radarRec.getDataURI(), data); - dataStore.addDataRecord(bdr, sp); - } + if (radarRec.getMapRecordVals() != null) { + byte[] data = DynamicSerializationManager.getManager( + SerializationType.Thrift).serialize( + radarRec.getMapRecordVals()); + ByteDataRecord bdr = new ByteDataRecord( + RadarStoredData.RECORD_VALS_ID, radarRec.getDataURI(), data); + bdr.setCorrelationObject(radarRec); + dataStore.addDataRecord(bdr, sp); + } - if (radarRec.getStormIDs() != null) { - byte[] data = DynamicSerializationManager.getManager( - SerializationType.Thrift).serialize(radarRec.getStormIDs()); - ByteDataRecord bdr = new ByteDataRecord( - RadarStoredData.STORM_IDS_ID, radarRec.getDataURI(), data); - dataStore.addDataRecord(bdr, sp); - } + if (radarRec.getStormIDs() != null) { + byte[] data = DynamicSerializationManager.getManager( + SerializationType.Thrift).serialize(radarRec.getStormIDs()); + ByteDataRecord bdr = new ByteDataRecord( + RadarStoredData.STORM_IDS_ID, radarRec.getDataURI(), data); + bdr.setCorrelationObject(radarRec); + dataStore.addDataRecord(bdr, sp); + } - if (radarRec.getGsmMessage() != null) { - byte[] data = DynamicSerializationManager.getManager( - SerializationType.Thrift).serialize( - radarRec.getGsmMessage()); - ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.GSM_ID, - radarRec.getDataURI(), data); - dataStore.addDataRecord(bdr, sp); - } - if (radarRec.getAlertMessage() != null) { - byte[] data = DynamicSerializationManager.getManager( - SerializationType.Thrift).serialize( - radarRec.getAlertMessage()); - ByteDataRecord bdr = new ByteDataRecord( - RadarStoredData.ALERT_MESSAGE_ID, radarRec.getDataURI(), - data); - dataStore.addDataRecord(bdr, sp); - } + if (radarRec.getGsmMessage() != null) { + byte[] data = DynamicSerializationManager.getManager( + SerializationType.Thrift).serialize( + radarRec.getGsmMessage()); + ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.GSM_ID, + radarRec.getDataURI(), data); + bdr.setCorrelationObject(radarRec); + dataStore.addDataRecord(bdr, sp); + } + if (radarRec.getAlertMessage() != null) { + byte[] data = DynamicSerializationManager.getManager( + SerializationType.Thrift).serialize( + radarRec.getAlertMessage()); + ByteDataRecord bdr = new ByteDataRecord( + RadarStoredData.ALERT_MESSAGE_ID, radarRec.getDataURI(), + data); + bdr.setCorrelationObject(radarRec); + dataStore.addDataRecord(bdr, sp); + } - if (radarRec.getAapMessage() != null) { - byte[] data = DynamicSerializationManager.getManager( - SerializationType.Thrift).serialize( - radarRec.getAapMessage()); - ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.AAP_ID, - radarRec.getDataURI(), data); - dataStore.addDataRecord(bdr, sp); - } + if (radarRec.getAapMessage() != null) { + byte[] data = DynamicSerializationManager.getManager( + SerializationType.Thrift).serialize( + radarRec.getAapMessage()); + ByteDataRecord bdr = new ByteDataRecord(RadarStoredData.AAP_ID, + radarRec.getDataURI(), data); + bdr.setCorrelationObject(radarRec); + dataStore.addDataRecord(bdr, sp); + } - return dataStore; - } + return dataStore; + } - @Override - public List getHDF5Data(List objects, - int tileSet) throws PluginException { - List retVal = new ArrayList(); + @Override + public List getHDF5Data(List objects, + int tileSet) throws PluginException { + List retVal = new ArrayList(); - for (PluginDataObject obj : objects) { - IDataRecord[] record = null; + for (PluginDataObject obj : objects) { + IDataRecord[] record = null; - if (obj instanceof IPersistable) { - /* connect to the data store and retrieve the data */ - try { - record = getDataStore((IPersistable) obj).retrieve( - obj.getDataURI()); - } catch (Exception e) { - throw new PluginException( - "Error retrieving radar HDF5 data", e); - } - retVal.add(record); - } - } + if (obj instanceof IPersistable) { + /* connect to the data store and retrieve the data */ + try { + record = getDataStore((IPersistable) obj).retrieve( + obj.getDataURI()); + } catch (Exception e) { + throw new PluginException( + "Error retrieving radar HDF5 data", e); + } + retVal.add(record); + } + } - return retVal; - } + return retVal; + } - @Override - public PluginDataObject[] getFullRecord(DatabaseQuery query, int tile) - throws PluginException { - PluginDataObject[] queryResults = getMetadata(query); - for (PluginDataObject obj : queryResults) { - RadarRecord record = (RadarRecord) obj; - record.setPluginName(pluginName); - IDataRecord[] hdf5Data = getHDF5Data(record, tile); - record.setMessageData(hdf5Data[0].getDataObject()); - record.setAngleData((float[]) hdf5Data[1].getDataObject()); - record.setThresholds((short[]) hdf5Data[2].getDataObject()); - record.setProductDependentValues((short[]) hdf5Data[8] - .getDataObject()); + @Override + public PluginDataObject[] getFullRecord(DatabaseQuery query, int tile) + throws PluginException { + PluginDataObject[] queryResults = getMetadata(query); + for (PluginDataObject obj : queryResults) { + RadarRecord record = (RadarRecord) obj; + record.setPluginName(pluginName); + IDataRecord[] hdf5Data = getHDF5Data(record, tile); + record.setMessageData(hdf5Data[0].getDataObject()); + record.setAngleData((float[]) hdf5Data[1].getDataObject()); + record.setThresholds((short[]) hdf5Data[2].getDataObject()); + record.setProductDependentValues((short[]) hdf5Data[8] + .getDataObject()); - record.setProductVals((HashMap>>) hdf5Data[5] - .getDataObject()); - record.setMapRecordVals((HashMap>) hdf5Data[6]); - record.setGsmMessage((GSMMessage) hdf5Data[7].getDataObject()); - try { - record.setSymbologyBlock((SymbologyBlock) SerializationUtil - .transformFromThrift((byte[]) hdf5Data[3] - .getDataObject())); - record.setGraphicBlock((GraphicBlock) SerializationUtil - .transformFromThrift((byte[]) hdf5Data[4] - .getDataObject())); + record.setProductVals((HashMap>>) hdf5Data[5] + .getDataObject()); + record.setMapRecordVals((HashMap>) hdf5Data[6]); + record.setGsmMessage((GSMMessage) hdf5Data[7].getDataObject()); + try { + record.setSymbologyBlock((SymbologyBlock) SerializationUtil + .transformFromThrift((byte[]) hdf5Data[3] + .getDataObject())); + record.setGraphicBlock((GraphicBlock) SerializationUtil + .transformFromThrift((byte[]) hdf5Data[4] + .getDataObject())); - } catch (SerializationException e) { - throw new PluginException( - "Error deserializing symbology block", e); - } - } - return queryResults; - } + } catch (SerializationException e) { + throw new PluginException( + "Error deserializing symbology block", e); + } + } + return queryResults; + } - public void populateData(RadarRecord record) throws Exception { - RadarDataRetriever.populateRadarRecord(getDataStore(record), record); - } + public void populateData(RadarRecord record) throws Exception { + RadarDataRetriever.populateRadarRecord(getDataStore(record), record); + } } diff --git a/edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/PyPiesDataStore.java b/edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/PyPiesDataStore.java index 64c6a7a498..94927e3bb8 100644 --- a/edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/PyPiesDataStore.java +++ b/edexOsgi/com.raytheon.uf.common.pypies/src/com/raytheon/uf/common/pypies/PyPiesDataStore.java @@ -3,10 +3,12 @@ package com.raytheon.uf.common.pypies; import java.io.File; import java.io.FileNotFoundException; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.Map; import com.raytheon.uf.common.comm.HttpClient; +import com.raytheon.uf.common.datastorage.DuplicateRecordStorageException; import com.raytheon.uf.common.datastorage.IDataStore; import com.raytheon.uf.common.datastorage.Request; import com.raytheon.uf.common.datastorage.StorageException; @@ -277,13 +279,37 @@ public class PyPiesDataStore implements IDataStore { StorageStatus ss = null; try { StoreResponse sr = (StoreResponse) sendRequest(req); - records.clear(); ss = sr.getStatus(); String[] exc = sr.getExceptions(); IDataRecord[] failed = sr.getFailedRecords(); + + // need to set the correlation object + if (failed != null) { + for (IDataRecord rec : failed) { + Iterator recordIter = records.iterator(); + while (recordIter.hasNext()) { + IDataRecord oldRec = recordIter.next(); + if (oldRec.getGroup().equals(rec.getGroup()) + && oldRec.getName().equals(rec.getName())) { + rec.setCorrelationObject(oldRec + .getCorrelationObject()); + recordIter.remove(); + break; + } + } + } + } + + records.clear(); StorageException[] jexc = new StorageException[exc.length]; for (int i = 0; i < exc.length; i++) { - jexc[i] = new StorageException(exc[i], failed[i]); + // checking for duplicates based on what is in the string... + if (exc[i].contains("already exists")) { + jexc[i] = new DuplicateRecordStorageException(exc[i], + failed[i]); + } else { + jexc[i] = new StorageException(exc[i], failed[i]); + } } ss.setExceptions(jexc);