Issue #2613 Registry performance enhancements. Better handling of delete events.

Change-Id: I56d3f5c37778e717810b6a698ec9405ed3cde00a

Former-commit-id: 95b0f12880 [formerly 216651e387 [formerly b53a0b258d] [formerly 95b0f12880 [formerly f863897fd512acae11364eb6b4737339b5951da0]]]
Former-commit-id: 216651e387 [formerly b53a0b258d]
Former-commit-id: 216651e387
Former-commit-id: eac441fe9d
This commit is contained in:
Benjamin Phillippe 2014-01-21 13:00:31 -06:00
parent 8870950160
commit 0d9c491e77
12 changed files with 239 additions and 47 deletions

View file

@ -77,6 +77,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* among multiple ref lists
* 12/2/2013 1829 bphillip Modified persistence annotations, added
* constructors, hashCode, toString and equals
* 01/21/2014 2613 bphillip Added equals and hashcode
* </pre>
*
* @author bphillip
@ -144,6 +145,31 @@ public class ObjectRefType implements IPersistableDataObject<String> {
this.key = key;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((id == null) ? 0 : id.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
ObjectRefType other = (ObjectRefType) obj;
if (id == null) {
if (other.id != null)
return false;
} else if (!id.equals(other.id))
return false;
return true;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();

View file

@ -22,7 +22,6 @@
<constructor-arg>
<bean
class="com.raytheon.uf.common.datadelivery.registry.handlers.EmptyAdhocSubscriptionHandler">
<property name="registryHandler" ref="registryHandler" />
</bean>
</constructor-arg>
</bean>

View file

@ -24,7 +24,6 @@
value="com.raytheon.uf.edex.datadelivery.registry.federation.RegistryFederationManager.addObjectTypesToSubscribeTo" />
<property name="arguments">
<list>
<value>Test Object Type</value>
<value>urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:Federation</value>
<value>urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:Registry</value>
<value>urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:Association</value>

View file

@ -156,6 +156,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.EbxmlObjectUtil;
* 12/2/2013 1829 bphillip Modified to use correct getters for slot values
* 12/9/2013 2613 bphillip Optimized registry sync function
* 1/15/2014 2613 bphillip Added leaveFederation method to prevent inactive registries from participating in the federation unintentionally.
* 1/21/2014 2613 bphillip Changed max down time which requires a sync
* </pre>
*
* @author bphillip
@ -188,8 +189,7 @@ public class RegistryFederationManager implements RegistryInitializedListener {
* The maximum time a registry can be down before a full synchronization is
* performed
*/
private static final long MAX_DOWN_TIME_DURATION = TimeUtil.MILLIS_PER_DAY
* 2 - TimeUtil.MILLIS_PER_HOUR;
private static final long MAX_DOWN_TIME_DURATION = TimeUtil.MILLIS_PER_HOUR * 6;
/** The central registry mode string */
private static final String CENTRAL_REGISTRY_MODE = "centralRegistry";

View file

@ -74,6 +74,7 @@
<bean name="AuditableEventTypeDao"
class="com.raytheon.uf.edex.registry.ebxml.dao.AuditableEventTypeDao">
<property name="sessionFactory" ref="metadataSessionFactory" />
<property name="soapService" ref="registryWebServiceClient"/>
</bean>
<bean name="classificationNodeDao"

View file

@ -20,21 +20,32 @@
package com.raytheon.uf.edex.registry.ebxml.dao;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
import javax.xml.datatype.XMLGregorianCalendar;
import oasis.names.tc.ebxml.regrep.wsdl.registry.services.v4.MsgRegistryException;
import oasis.names.tc.ebxml.regrep.xsd.query.v4.QueryRequest;
import oasis.names.tc.ebxml.regrep.xsd.query.v4.ResponseOptionType;
import oasis.names.tc.ebxml.regrep.xsd.rim.v4.ActionType;
import oasis.names.tc.ebxml.regrep.xsd.rim.v4.AuditableEventType;
import oasis.names.tc.ebxml.regrep.xsd.rim.v4.DateTimeValueType;
import oasis.names.tc.ebxml.regrep.xsd.rim.v4.ObjectRefType;
import oasis.names.tc.ebxml.regrep.xsd.rim.v4.SlotType;
import oasis.names.tc.ebxml.regrep.xsd.rim.v4.SubscriptionType;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import com.raytheon.uf.common.registry.constants.ActionTypes;
import com.raytheon.uf.common.registry.constants.QueryReturnTypes;
import com.raytheon.uf.common.registry.services.RegistrySOAPServices;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.common.util.CollectionUtil;
import com.raytheon.uf.edex.registry.ebxml.exception.EbxmlRegistryException;
@ -58,6 +69,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.EbxmlObjectUtil;
* 9/11/2013 2354 bphillip Modified queries to find deleted objects
* 10/23/2013 1538 bphillip Changed send time slot to be DateTimeValue instead of integer
* 12/2/2013 1829 bphillip Changed to use non generic getter of value type
* 01/21/2014 2613 bphillip Modified queries to better handle deletes
*
* </pre>
*
@ -79,14 +91,14 @@ public class AuditableEventTypeDao extends
+ "left outer join action.affectedObjectRefs as AffectedObjectRefs "
+ "left outer join AffectedObjects.registryObject as RegistryObjects "
+ "left outer join AffectedObjectRefs.objectRef as ObjRefs "
+ "where (ObjRefs.id in (:ids) OR RegistryObjects.id in (:ids) OR action.eventType = :eventType) and event.timestamp >= :startTime";
+ "where (ObjRefs.id in (:ids) OR RegistryObjects.id in (:ids)) and event.timestamp >= :startTime";
/**
* Query to find deleted events
*/
private static final String FIND_DELETED_EVENTS_OF_INTEREST_QUERY = "select event from AuditableEventType as event "
+ "left outer join event.action as action "
+ "where action.eventType = :eventType and event.timestamp >= :startTime";
+ "where action.eventType = 'urn:oasis:names:tc:ebxml-regrep:ActionType:delete' and event.timestamp > :startTime";
/** Optional end time clause */
private static final String END_TIME_CLAUSE = " and event.timestamp <= :endTime";
@ -104,6 +116,17 @@ public class AuditableEventTypeDao extends
private static final String GET_EXPIRED_EVENTS_QUERY = "FROM AuditableEventType event where event.timestamp < :"
+ GET_EXPIRED_EVENTS_QUERY_CUTOFF_PARAMETER;
/** The registry soap services */
private RegistrySOAPServices soapService;
/** Sorter for sorting events */
private static final Comparator<AuditableEventType> EVENT_TIME_COMPARATOR = new Comparator<AuditableEventType>() {
@Override
public int compare(AuditableEventType o1, AuditableEventType o2) {
return o2.getTimestamp().compare(o1.getTimestamp());
}
};
/**
* Constructor.
*
@ -134,6 +157,100 @@ public class AuditableEventTypeDao extends
.getTimeInMillis()));
}
/**
* Gets all auditable events which reference the objects of interest.
*
* @param subscription
* The subscription to get the events for
* @param serviceAddress
* The address to the registry to use to verify deleted objects
* @param startTime
* The start time boundary of the query
* @param endTime
* The end time boundary of the query
* @param objectsOfInterest
* The objects of interest to get events for
* @return The list of auditable events referencing the objects of interest
* @throws EbxmlRegistryException
* @throws MsgRegistryException
*/
public List<AuditableEventType> getEventsOfInterest(
SubscriptionType subscription, String serviceAddress,
XMLGregorianCalendar startTime, XMLGregorianCalendar endTime,
List<ObjectRefType> objectsOfInterest)
throws EbxmlRegistryException, MsgRegistryException {
List<AuditableEventType> events = new ArrayList<AuditableEventType>(0);
if (!objectsOfInterest.isEmpty()) {
events = getEventsOfInterest(FIND_EVENTS_OF_INTEREST_QUERY,
startTime, endTime, objectsOfInterest);
}
List<AuditableEventType> deleteEvents = getDeleteEventsOfInterest(
subscription, serviceAddress, startTime, endTime);
if (!deleteEvents.isEmpty()) {
events.addAll(deleteEvents);
}
Collections.sort(events, EVENT_TIME_COMPARATOR);
return events;
}
/**
* Gets applicable delete events
*
* @param subscription
* The subscription to get the events for
* @param serviceAddress
* The address to the registry to use to verify deleted objects
* @param startTime
* The start time boundary of the query
* @param endTime
* The end time boundary of the query
* @return The list of auditable events referencing deleted objects
* @throws EbxmlRegistryException
* @throws MsgRegistryException
*/
private List<AuditableEventType> getDeleteEventsOfInterest(
SubscriptionType subscription, String serviceAddress,
XMLGregorianCalendar startTime, XMLGregorianCalendar endTime)
throws EbxmlRegistryException, MsgRegistryException {
List<AuditableEventType> retVal = new LinkedList<AuditableEventType>();
List<AuditableEventType> deletedEvents = getEventsOfInterest(
FIND_DELETED_EVENTS_OF_INTEREST_QUERY, startTime, endTime, null);
try {
URL url = new URL(serviceAddress);
String baseURL = url.toString().replace(url.getPath(), "");
List<ObjectRefType> remoteRefs = soapService
.getQueryServiceForHost(baseURL)
.executeQuery(
new QueryRequest(
"Deleted Objects of Interest Query for ["
+ subscription.getId() + "]",
subscription.getSelector(),
new ResponseOptionType(
QueryReturnTypes.OBJECT_REF, false)))
.getObjectRefList().getObjectRef();
for (AuditableEventType event : deletedEvents) {
for (ActionType action : event.getAction()) {
if (action.getAffectedObjectRefs() != null
&& !action.getAffectedObjectRefs().getObjectRef()
.isEmpty()) {
if (remoteRefs.contains(action.getAffectedObjectRefs()
.getObjectRef().get(0))) {
retVal.add(event);
}
}
}
}
} catch (MalformedURLException e) {
throw new EbxmlRegistryException(
"Error parsing notification address", e);
}
return retVal;
}
/**
* Gets the events of interest based on the start time, end time, and the
* list of objects of interest
@ -147,13 +264,10 @@ public class AuditableEventTypeDao extends
* @return The list of auditable events of interest within the constrains of
* the start time, end time and including the objects of interest
*/
public List<AuditableEventType> getEventsOfInterest(
private List<AuditableEventType> getEventsOfInterest(String query,
XMLGregorianCalendar startTime, XMLGregorianCalendar endTime,
List<ObjectRefType> objectsOfInterest) {
String query = FIND_DELETED_EVENTS_OF_INTEREST_QUERY;
List<Object> queryParams = new ArrayList<Object>(4);
queryParams.add("eventType");
queryParams.add(ActionTypes.delete);
List<Object> queryParams = new ArrayList<Object>(2);
queryParams.add("startTime");
queryParams.add(startTime);
if (!CollectionUtil.isNullOrEmpty(objectsOfInterest)) {
@ -241,4 +355,8 @@ public class AuditableEventTypeDao extends
return AuditableEventType.class;
}
public void setSoapService(RegistrySOAPServices soapService) {
this.soapService = soapService;
}
}

View file

@ -19,6 +19,7 @@
**/
package com.raytheon.uf.edex.registry.ebxml.services;
import java.util.ArrayList;
import java.util.List;
import oasis.names.tc.ebxml.regrep.xsd.rim.v4.ActionType;
@ -34,6 +35,7 @@ import oasis.names.tc.ebxml.regrep.xsd.rs.v4.RegistryRequestType;
import org.springframework.transaction.annotation.Transactional;
import com.google.common.eventbus.Subscribe;
import com.raytheon.uf.common.registry.constants.ActionTypes;
import com.raytheon.uf.common.registry.constants.RegistryObjectTypes;
import com.raytheon.uf.common.registry.constants.StatusTypes;
import com.raytheon.uf.common.registry.ebxml.RegistryUtil;
@ -59,6 +61,7 @@ import com.raytheon.uf.edex.registry.events.CreateAuditTrailEvent;
* 10/23/2013 1538 bphillip Removed call to subscription manager. Subscriptions will now
* only be run on a quartz timer
* 12/2/2013 1829 bphillip Now uses event bus for triggering auditable event generation
* 01/21/2014 2613 bphillip Changed how auditable events are created for deletes
*
* </pre>
*
@ -103,12 +106,27 @@ public class AuditableEventService {
public void createAuditableEventFromObjects(
CreateAuditTrailEvent registryEvent) throws EbxmlRegistryException {
if (!CollectionUtil.isNullOrEmpty(registryEvent.getObjectsAffected())) {
AuditableEventType event = createEvent(registryEvent.getRequest(),
TimeUtil.currentTimeMillis());
addRegistryObjectActionToEvent(event,
registryEvent.getActionType(),
registryEvent.getObjectsAffected());
auditDao.createOrUpdate(event);
long currentTime = TimeUtil.currentTimeMillis();
if (ActionTypes.delete.equals(registryEvent.getActionType())) {
for (RegistryObjectType obj : registryEvent
.getObjectsAffected()) {
List<RegistryObjectType> regObjList = new ArrayList<RegistryObjectType>(
1);
regObjList.add(obj);
AuditableEventType event = createEvent(
registryEvent.getRequest(), currentTime);
addRegistryObjectActionToEvent(event,
registryEvent.getActionType(), regObjList);
auditDao.createOrUpdate(event);
}
} else {
AuditableEventType event = createEvent(
registryEvent.getRequest(), currentTime);
addRegistryObjectActionToEvent(event,
registryEvent.getActionType(),
registryEvent.getObjectsAffected());
auditDao.createOrUpdate(event);
}
}
}

View file

@ -107,6 +107,7 @@ import com.raytheon.uf.edex.registry.events.CreateAuditTrailEvent;
* Nov 08, 2013 2506 bgonzale Added RegistryObjectType to RemoveRegistryEvent.
* Separate update from create notifications.
* 12/2/2013 1829 bphillip Auditable events are not genereted via messages on the event bus
* 01/21/2014 2613 bphillip Removed verbose log message from removeObjects
*
*
* </pre>
@ -230,8 +231,6 @@ public class LifecycleManagerImpl implements LifecycleManager {
statusHandler
.info("No results returned from remove objects query");
} else {
statusHandler.info("Remove objects query returned "
+ queryResponse.getRegistryObjects() + " objects");
objectsToRemove.addAll(queryResponse.getRegistryObjects());
}
}

View file

@ -92,6 +92,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.EbxmlObjectUtil;
* 10/30/2013 1538 bphillip Changed to use non-static registry soap service client
* 12/2/2013 1829 bphillip Added getIdsFrom action method and changed how slots are added to objects
* 1/15/2014 2613 bphillip Added batching of notification update queries to reduce number of web service calls
* 01/21/2014 2613 bphillip Added home slot to remove objects request so delete events are properly handled
*
* </pre>
*
@ -192,6 +193,9 @@ public class NotificationListenerImpl implements NotificationListener {
"Notification delete object submission", null,
null, refList, false, true,
DeletionScope.DELETE_ALL);
request.getSlot().add(
new SlotType(EbxmlObjectUtil.HOME_SLOT_NAME,
new StringValueType(clientBaseURL)));
try {
lcm.removeObjects(request);
} catch (MsgRegistryException e) {

View file

@ -74,6 +74,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.EbxmlObjectUtil;
* 10/23/2013 1538 bphillip Adding log messages and changed methods to handle DateTime value on
* AuditableEvents instead of integer
* 12/9/2013 2613 bphillip Changed start time boundary of get auditable events to be the last run time of the subscription
* 01/21/2014 2613 bphillip Changed start time boundary again and also a few minor cleanup items
* </pre>
*
* @author bphillip
@ -136,12 +137,16 @@ public class RegistryNotificationManager {
* @param objectsOfInterest
* The objects to get events for
* @return The events of interest for the given set of objects
* @throws MsgRegistryException
* @throws EbxmlRegistryException
*/
public List<AuditableEventType> getEventsOfInterest(
SubscriptionType subscription, String serviceAddress,
XMLGregorianCalendar startTime, XMLGregorianCalendar endTime,
List<ObjectRefType> objectsOfInterest) {
return this.auditableEventDao.getEventsOfInterest(startTime, endTime,
objectsOfInterest);
List<ObjectRefType> objectsOfInterest)
throws EbxmlRegistryException, MsgRegistryException {
return this.auditableEventDao.getEventsOfInterest(subscription,
serviceAddress, startTime, endTime, objectsOfInterest);
}
/**
@ -221,28 +226,26 @@ public class RegistryNotificationManager {
* If errors occur while sending the notifications
* @throws MsgRegistryException
*/
protected void sendNotifications(
SubscriptionNotificationListeners notificationListeners)
throws EbxmlRegistryException, MsgRegistryException {
protected XMLGregorianCalendar sendNotifications(
SubscriptionNotificationListeners notificationListeners,
XMLGregorianCalendar startTime) throws EbxmlRegistryException,
MsgRegistryException {
// Object to hold the last timestampe of the latest event in order to
// update the subscription last run time correctly
XMLGregorianCalendar lastTime = null;
final List<NotificationListenerWrapper> listeners = notificationListeners.listeners;
final SubscriptionType subscription = notificationListeners.subscription;
List<ObjectRefType> objectsOfInterest = getObjectsOfInterest(subscription);
XMLGregorianCalendar startTime = subscription
.getSlotValue(EbxmlObjectUtil.SUBSCRIPTION_LAST_RUN_TIME_SLOT_NAME);
if (startTime == null) {
startTime = subscription.getStartTime();
}
List<AuditableEventType> eventsOfInterest = getEventsOfInterest(
subscription.getStartTime(), subscription.getEndTime(),
objectsOfInterest);
if (!eventsOfInterest.isEmpty()) {
for (NotificationListenerWrapper listener : listeners) {
for (NotificationListenerWrapper listener : listeners) {
List<AuditableEventType> eventsOfInterest = getEventsOfInterest(
subscription, listener.address, startTime,
subscription.getEndTime(), objectsOfInterest);
if (!eventsOfInterest.isEmpty()) {
lastTime = eventsOfInterest.get(eventsOfInterest.size() - 1)
.getTimestamp();
int subListCount = eventsOfInterest.size()
/ notificationBatchSize;
int lastListSize = eventsOfInterest.size()
@ -281,6 +284,7 @@ public class RegistryNotificationManager {
}
}
return lastTime;
}
/**
@ -326,23 +330,28 @@ public class RegistryNotificationManager {
List<ActionType> actionList = event.getAction();
for (ActionType action : actionList) {
objectsToRemove.clear();
refsToRemove.clear();
if (action.getAffectedObjectRefs() != null) {
List<ObjectRefType> objRefs = action
.getAffectedObjectRefs().getObjectRef();
for (ObjectRefType obj : objRefs) {
boolean found = objectInList(objectsOfInterest, obj);
if (!found && !action.equals(ActionTypes.delete)) {
if (!found
&& !action.getEventType().equals(
ActionTypes.delete)) {
refsToRemove.add(obj);
}
}
objRefs.removeAll(objectsToRemove);
objRefs.removeAll(refsToRemove);
} else if (action.getAffectedObjects() != null) {
List<RegistryObjectType> regObjs = action
.getAffectedObjects().getRegistryObject();
for (RegistryObjectType obj : regObjs) {
boolean found = objectInList(objectsOfInterest, obj);
if (!found && !action.equals(ActionTypes.delete)) {
if (!found
&& !action.getEventType().equals(
ActionTypes.delete)) {
objectsToRemove.add(obj);
}
}

View file

@ -81,6 +81,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.EbxmlObjectUtil;
* 11/20/2013 2534 bphillip Moved method to get notification destinations to utility
* 12/9/2013 2613 bphillip Setting last run time of subscription now occurs before notification is sent
* 1/15/2014 2613 bphillip Added Hibernate flush and clear after subscription processing
* 01/21/2014 2613 bphillip Changed how last run time is updated for replication subscriptions
* </pre>
*
* @author bphillip
@ -388,9 +389,19 @@ public class RegistrySubscriptionManager implements
}
statusHandler.info("Processing subscription [" + subscriptionName
+ "]...");
updateLastRunTime(subscription, TimeUtil.currentTimeMillis());
notificationManager.sendNotifications(listeners
.get(subscriptionName));
XMLGregorianCalendar startTime = subscription
.getSlotValue(EbxmlObjectUtil.SUBSCRIPTION_LAST_RUN_TIME_SLOT_NAME);
if (startTime == null) {
startTime = subscription.getStartTime();
}
XMLGregorianCalendar lastEventTime = notificationManager
.sendNotifications(listeners.get(subscriptionName),
startTime);
if (lastEventTime != null) {
updateLastRunTime(subscription, lastEventTime
.toGregorianCalendar().getTimeInMillis());
}
} catch (Throwable e) {
statusHandler.error(
"Errors occurred while processing subscription ["

View file

@ -64,6 +64,7 @@ import com.raytheon.uf.edex.registry.ebxml.util.EbxmlObjectUtil;
* Jun 24, 2013 2106 djohnson Requires a transaction to be open, will not create one.
* 10/8/2013 1682 bphillip Refactored querying
* 11/20/2013 2534 bphillip Changed call to getNotificationDestinations which is not in a utility class
* 01/21/2014 2613 bphillip Modifications to account for changed method signatures in RegistryNotificationManager
*
* </pre>
*
@ -135,8 +136,15 @@ public class GetNotification extends RegistryQueryPlugin {
List<ObjectRefType> objectsOfInterest = notificationManager
.getObjectsOfInterest(subscription);
List<AuditableEventType> eventsOfInterest = notificationManager
.getEventsOfInterest(startTime, null, objectsOfInterest);
List<AuditableEventType> eventsOfInterest = null;
try {
eventsOfInterest = notificationManager.getEventsOfInterest(
subscription, destinations.get(0).getDestination(),
startTime, null, objectsOfInterest);
} catch (EbxmlRegistryException e1) {
throw EbxmlExceptionUtil.createMsgRegistryException(
"Error getting events!", e1);
}
try {
return createResponse(Arrays.asList(notificationManager
.getNotification(subscription, "Test Address",