From dc369018c35389e09cf6965b4d47b0cd5379f61d Mon Sep 17 00:00:00 2001 From: Nate Jensen Date: Wed, 11 Sep 2013 15:40:34 -0500 Subject: [PATCH] Issue #2359 initial checkin of remote log service Change-Id: Idd48062f204002956f63c6d449868a7bd9695263 Former-commit-id: 74c1e60ad29ca81b2f59c2c3eefff3d55db4ed9e [formerly 3162ae260b6a2d7096099cb31c604279ec459bc3] [formerly f148663bd1fbd6d16d9e9c0133fd298ebf99d653 [formerly 12988c63aeccc678f25f4210ad484a080ddd8aa7]] Former-commit-id: f148663bd1fbd6d16d9e9c0133fd298ebf99d653 Former-commit-id: ebf89475c591b1a6384559d053dadd134ad0c7d0 --- .../com.raytheon.uf.logsrv/.classpath | 7 + javaUtilities/com.raytheon.uf.logsrv/.project | 28 + .../.settings/org.eclipse.jdt.core.prefs | 7 + .../META-INF/MANIFEST.MF | 15 + .../com.raytheon.uf.logsrv/build.properties | 4 + .../com.raytheon.uf.logsrv/config.xml | 11 + .../com.raytheon.uf.logsrv/derby.log | 13 + .../com.raytheon.uf.logsrv/receiver.xml | 48 ++ .../com/raytheon/uf/logsrv/LogService.java | 94 +++ .../uf/logsrv/LogServiceException.java | 73 +++ .../src/com/raytheon/uf/logsrv/StoredMsg.java | 169 ++++++ .../uf/logsrv/config/LogSrvConfig.java | 143 +++++ .../uf/logsrv/derby/DerbyAppender.java | 58 ++ .../raytheon/uf/logsrv/derby/DerbyDao.java | 536 ++++++++++++++++++ .../uf/logsrv/quartz/CountRowsJob.java | 69 +++ .../uf/logsrv/quartz/CreateSendReportJob.java | 103 ++++ .../uf/logsrv/quartz/JobScheduler.java | 75 +++ .../uf/logsrv/report/TestReportOutputter.java | 67 +++ .../report/data/LogReportContainer.java | 90 +++ .../uf/logsrv/report/data/LogReportEvent.java | 184 ++++++ .../uf/logsrv/report/email/HtmlGenerator.java | 194 +++++++ .../uf/logsrv/report/email/ReportEmailer.java | 83 +++ 22 files changed, 2071 insertions(+) create mode 100644 javaUtilities/com.raytheon.uf.logsrv/.classpath create mode 100644 javaUtilities/com.raytheon.uf.logsrv/.project create mode 100644 javaUtilities/com.raytheon.uf.logsrv/.settings/org.eclipse.jdt.core.prefs create mode 100644 javaUtilities/com.raytheon.uf.logsrv/META-INF/MANIFEST.MF create mode 100644 javaUtilities/com.raytheon.uf.logsrv/build.properties create mode 100644 javaUtilities/com.raytheon.uf.logsrv/config.xml create mode 100644 javaUtilities/com.raytheon.uf.logsrv/derby.log create mode 100644 javaUtilities/com.raytheon.uf.logsrv/receiver.xml create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/LogService.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/LogServiceException.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/StoredMsg.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/config/LogSrvConfig.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/derby/DerbyAppender.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/derby/DerbyDao.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/quartz/CountRowsJob.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/quartz/CreateSendReportJob.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/quartz/JobScheduler.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/TestReportOutputter.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/data/LogReportContainer.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/data/LogReportEvent.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/email/HtmlGenerator.java create mode 100644 javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/email/ReportEmailer.java diff --git a/javaUtilities/com.raytheon.uf.logsrv/.classpath b/javaUtilities/com.raytheon.uf.logsrv/.classpath new file mode 100644 index 0000000000..ad32c83a78 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/javaUtilities/com.raytheon.uf.logsrv/.project b/javaUtilities/com.raytheon.uf.logsrv/.project new file mode 100644 index 0000000000..5c9e71cfba --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/.project @@ -0,0 +1,28 @@ + + + com.raytheon.uf.logsrv + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/javaUtilities/com.raytheon.uf.logsrv/.settings/org.eclipse.jdt.core.prefs b/javaUtilities/com.raytheon.uf.logsrv/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000000..c537b63063 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,7 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/javaUtilities/com.raytheon.uf.logsrv/META-INF/MANIFEST.MF b/javaUtilities/com.raytheon.uf.logsrv/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..b48d5ea02d --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/META-INF/MANIFEST.MF @@ -0,0 +1,15 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Logsrv +Bundle-SymbolicName: com.raytheon.uf.logsrv +Bundle-Version: 1.0.0.qualifier +Bundle-Activator: com.raytheon.uf.logsrv.Activator +Bundle-Vendor: RAYTHEON +Bundle-RequiredExecutionEnvironment: JavaSE-1.6 +Bundle-ActivationPolicy: lazy +Require-Bundle: ch.qos.logback;bundle-version="1.0.13", + org.slf4j;bundle-version="1.7.5", + org.apache.derby;bundle-version="10.10.1", + javax.mail, + org.quartz;bundle-version="1.8.6", + org.apache.commons.lang;bundle-version="2.3.0" diff --git a/javaUtilities/com.raytheon.uf.logsrv/build.properties b/javaUtilities/com.raytheon.uf.logsrv/build.properties new file mode 100644 index 0000000000..34d2e4d2da --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/build.properties @@ -0,0 +1,4 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + . diff --git a/javaUtilities/com.raytheon.uf.logsrv/config.xml b/javaUtilities/com.raytheon.uf.logsrv/config.xml new file mode 100644 index 0000000000..bc353200bb --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/config.xml @@ -0,0 +1,11 @@ + + + ts1-oma + /awips2/edex/data/utility/nate + Nathan.Jensen@raytheon.com + mk2-msg10.raymail.ray.com + 143 + + Nathan.Jensen@raytheon.com + 01:30 + \ No newline at end of file diff --git a/javaUtilities/com.raytheon.uf.logsrv/derby.log b/javaUtilities/com.raytheon.uf.logsrv/derby.log new file mode 100644 index 0000000000..ae8e4c44f6 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/derby.log @@ -0,0 +1,13 @@ +---------------------------------------------------------------- +Wed Sep 11 15:37:35 CDT 2013: +Booting Derby version The Apache Software Foundation - Apache Derby - 10.10.1.1 - (1458268): instance a816c00e-0141-0ebe-4f2c-000009e7d350 +on database directory /awips2/edex/data/utility/nate with class loader sun.misc.Launcher$AppClassLoader@4aad3ba4 +Loaded from file:/common/njensen/git/development/cots/org.apache.derby/derby.jar +java.vendor=Sun Microsystems Inc. +java.runtime.version=1.6.0_43-b01 +user.dir=/common/njensen/git/development/javaUtilities/com.raytheon.uf.logsrv +os.name=Linux +os.arch=amd64 +os.version=2.6.18-238.el5 +derby.system.home=null +Database Class Loader started - derby.database.classpath='' diff --git a/javaUtilities/com.raytheon.uf.logsrv/receiver.xml b/javaUtilities/com.raytheon.uf.logsrv/receiver.xml new file mode 100644 index 0000000000..478777db3c --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/receiver.xml @@ -0,0 +1,48 @@ + + + + + + + + /home/njensen/logs/logService-internal-%d{yyyyMMdd}.log + 30 + + + %-5p %d [%t] %c{0}: %m%n + + + + + + %-5p %d [%t] %c{0}: %m%n + + + + + + + + + + + + + + + + + + + + + + + +
dev33.oma.us.ray.com
+ 5477 + + + + + \ No newline at end of file diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/LogService.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/LogService.java new file mode 100644 index 0000000000..c8ebe31790 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/LogService.java @@ -0,0 +1,94 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv; + +import java.io.File; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Unmarshaller; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; + +import com.raytheon.uf.logsrv.config.LogSrvConfig; +import com.raytheon.uf.logsrv.derby.DerbyDao; +import com.raytheon.uf.logsrv.quartz.JobScheduler; + +/** + * The main class of the logging service that loads the config files and + * therefore listens for incoming logging events. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 27, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class LogService { + + private static final String LOGBACK_CONFIG = "receiver.xml"; + + private static final String SERVICE_CONFIG = "config.xml"; + + private static final Logger logger = LoggerFactory + .getLogger("InternalLogger"); + + /** + * @param args + * @throws Exception + */ + public static void main(String[] args) throws Exception { + logger.info("Starting log analytics service"); + + JAXBContext context = JAXBContext.newInstance(LogSrvConfig.class); + Unmarshaller m = context.createUnmarshaller(); + LogSrvConfig config = (LogSrvConfig) m.unmarshal(new File( + SERVICE_CONFIG)); + config.validate(); + DerbyDao.setConfig(config); + logger.info("Logging events from " + config.getClusterName()); + + logger.info("Starting socket listener"); + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + lc.reset(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + configurator.doConfigure(LOGBACK_CONFIG); + + logger.info("Scheduling report generation"); + JobScheduler.scheduleJobs(config); + } + + public static Logger getLogger() { + return logger; + } + +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/LogServiceException.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/LogServiceException.java new file mode 100644 index 0000000000..4acd3f1b7f --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/LogServiceException.java @@ -0,0 +1,73 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv; + +/** + * An exception in the logging service. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 27, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class LogServiceException extends Exception { + + private static final long serialVersionUID = 1L; + + /** + * Default Constructor + * + */ + public LogServiceException() { + super(); + } + + /** + * @param message + */ + public LogServiceException(String message) { + super(message); + } + + /** + * @param message + * @param cause + */ + public LogServiceException(String message, Throwable cause) { + super(message, cause); + } + + /** + * @param cause + */ + public LogServiceException(Throwable cause) { + super(cause); + } + +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/StoredMsg.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/StoredMsg.java new file mode 100644 index 0000000000..f0c15b7ff9 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/StoredMsg.java @@ -0,0 +1,169 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv; + +import java.util.Date; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.StackTraceElementProxy; +import ch.qos.logback.classic.spi.ThrowableProxyUtil; + +/** + * A translation of a logging event to a POJO that closely resembles what is in + * the database. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 27, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class StoredMsg { + + private Date eventTime; + + private String message; + + private String javaClass; + + private int lineNumber; + + // TODO build date or build number? not sure how to get logback to send that + // along + + private String stacktrace; + + private String threadName; + + private String machineName; + + private String level; + + public StoredMsg(ILoggingEvent event) { + eventTime = new Date(event.getTimeStamp()); + + // find the most underlying cause + IThrowableProxy cause = event.getThrowableProxy(); + if (cause != null) { + stacktrace = ThrowableProxyUtil.asString(cause); + while (cause.getCause() != null) { + cause = cause.getCause(); + } + + message = cause.getMessage(); + StackTraceElementProxy[] stack = cause + .getStackTraceElementProxyArray(); + if (stack != null && stack.length > 0) { + StackTraceElement ste = stack[0].getStackTraceElement(); + lineNumber = ste.getLineNumber(); + javaClass = ste.getClassName(); + } + } else { + message = event.getMessage(); + } + threadName = event.getThreadName(); + machineName = event.getLoggerContextVO().getPropertyMap() + .get("HOSTNAME"); + level = event.getLevel().toString(); + + } + + public Date getEventTime() { + return eventTime; + } + + public void setEventTime(Date eventTime) { + this.eventTime = eventTime; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String getJavaClass() { + return javaClass; + } + + public void setJavaClass(String javaClass) { + this.javaClass = javaClass; + } + + public int getLineNumber() { + return lineNumber; + } + + public void setLineNumber(int lineNumber) { + this.lineNumber = lineNumber; + } + + public String getThreadName() { + return threadName; + } + + public void setThreadName(String threadName) { + this.threadName = threadName; + } + + public String getMachineName() { + return machineName; + } + + public void setMachineName(String machineName) { + this.machineName = machineName; + } + + public String getStacktrace() { + return stacktrace; + } + + public void setStacktrace(String stacktrace) { + this.stacktrace = stacktrace; + } + + public String getLevel() { + return level; + } + + public void setLevel(String level) { + this.level = level; + } + + @Override + public String toString() { + return "StoredMsg [eventTime=" + eventTime + ", message=" + message + + ", javaClass=" + javaClass + ", lineNumber=" + lineNumber + + ", stacktrace=" + stacktrace + ", threadName=" + threadName + + ", machineName=" + machineName + ", level=" + level + "]"; + } + +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/config/LogSrvConfig.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/config/LogSrvConfig.java new file mode 100644 index 0000000000..3e4bffca58 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/config/LogSrvConfig.java @@ -0,0 +1,143 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv.config; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.commons.lang.Validate; + +/** + * A configuration for the logging service. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 29, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +@XmlRootElement +@XmlAccessorType(XmlAccessType.NONE) +public class LogSrvConfig { + + @XmlElement() + private String clusterName; + + @XmlElement + private String databaseDir; + + @XmlElement + private String fromAddress; + + @XmlElement + private String smtpHost; + + @XmlElement + private int smtpPort; + + @XmlElement + private String toAddress; + + @XmlElement + private String timeToSend; + + public String getFromAddress() { + return fromAddress; + } + + public void setFromAddress(String fromAddress) { + this.fromAddress = fromAddress; + } + + public String getSmtpHost() { + return smtpHost; + } + + public void setSmtpHost(String smtpHost) { + this.smtpHost = smtpHost; + } + + public int getSmtpPort() { + return smtpPort; + } + + public void setSmtpPort(int smtpPort) { + this.smtpPort = smtpPort; + } + + public String getToAddress() { + return toAddress; + } + + public void setToAddress(String toAddress) { + this.toAddress = toAddress; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getTimeToSend() { + return timeToSend; + } + + public void setTimeToSend(String timeToSend) { + this.timeToSend = timeToSend; + } + + public String getDatabaseDir() { + return databaseDir; + } + + public void setDatabaseDir(String databaseDir) { + this.databaseDir = databaseDir; + } + + /** + * Validates that the config has every value set. + */ + public void validate() { + Validate.notEmpty(clusterName, "Config must include a clusterName"); + Validate.notEmpty(databaseDir, "Config must include a databaseDir"); + Validate.notEmpty(fromAddress, "Config must include a fromAddress"); + Validate.notEmpty(smtpHost, "Config must include an smtpHost"); + Validate.notEmpty(timeToSend, "Config must include a timeToSend"); + Validate.notEmpty(toAddress, "Config must include a toAddress"); + if (smtpPort <= 0) { + throw new IllegalArgumentException( + "Config must include an smtpPort"); + } + } + +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/derby/DerbyAppender.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/derby/DerbyAppender.java new file mode 100644 index 0000000000..31db60bad7 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/derby/DerbyAppender.java @@ -0,0 +1,58 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv.derby; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.AppenderBase; + +import com.raytheon.uf.logsrv.LogService; +import com.raytheon.uf.logsrv.LogServiceException; +import com.raytheon.uf.logsrv.StoredMsg; + +/** + * A logback appender that stores a logging event to the database. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 27, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class DerbyAppender extends AppenderBase { + + @Override + protected void append(ILoggingEvent eventObject) { + StoredMsg msg = new StoredMsg(eventObject); + try { + DerbyDao.getInstance().insert(msg); + } catch (LogServiceException e) { + LogService.getLogger().error( + "Error inserting message into derby database", e); + } + } +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/derby/DerbyDao.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/derby/DerbyDao.java new file mode 100644 index 0000000000..0f0159934b --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/derby/DerbyDao.java @@ -0,0 +1,536 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv.derby; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; + +import com.raytheon.uf.logsrv.LogService; +import com.raytheon.uf.logsrv.LogServiceException; +import com.raytheon.uf.logsrv.StoredMsg; +import com.raytheon.uf.logsrv.config.LogSrvConfig; +import com.raytheon.uf.logsrv.report.data.LogReportContainer; +import com.raytheon.uf.logsrv.report.data.LogReportEvent; + +/** + * DAO for interacting with the derby database to add rows or create report + * objects. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 27, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class DerbyDao { + + private static final String DRIVER = "org.apache.derby.jdbc.EmbeddedDriver"; + + private static final String TABLE = "log"; + + private static final String DB_CREATE = "CREATE TABLE " + TABLE + + "(pk INT NOT NULL GENERATED BY DEFAULT AS IDENTITY, " + + "eventTime TIMESTAMP, " + "message VARCHAR(32672), " + + "javaClass VARCHAR(128), " + "lineNumber INT, " + + "stacktrace VARCHAR(32672), " + "threadName VARCHAR(128), " + + "machineName VARCHAR(128), " + "level VARCHAR(32))"; + + private static final String INSERT_PREPARED_STATEMENT = "INSERT INTO " + + TABLE + + "(eventTime, message, javaClass, lineNumber, stacktrace, threadName, machineName, level) " + + "values (?, ?, ?, ?, ?, ?, ?, ?)"; + + private static final String PURGE_STATEMENT = "DELETE from " + TABLE; + + private static final String TIME_STATEMENT = "select min(eventTime), max(eventTime) FROM " + + TABLE; + + private static final String COUNT_STATEMENT = "select count(*) from " + + TABLE; + + private PreparedStatement saveStatement; + + private PreparedStatement purgeStatement; + + private PreparedStatement timeStatement; + + private PreparedStatement countStatement; + + private Connection connection; + + private static DerbyDao instance = new DerbyDao(); + + private static LogSrvConfig config; + + private DerbyDao() { + + } + + public static DerbyDao getInstance() { + return instance; + } + + public static void setConfig(LogSrvConfig cfg) { + config = cfg; + } + + /** + * Creates a connection to derby + * + * @return + * @throws LogServiceException + */ + private Connection createConnection() throws LogServiceException { + Statement statement = null; + boolean errorOccurred = false; + Connection connection = null; + try { + Class.forName(DRIVER).newInstance(); + connection = DriverManager.getConnection("jdbc:derby:directory:" + + config.getDatabaseDir() + ";create=true"); + DatabaseMetaData dmd = connection.getMetaData(); + ResultSet tables = dmd.getTables(null, null, null, + new String[] { "TABLE" }); + boolean needCreation = true; + + while (tables.next()) { + String tableName = tables.getString("TABLE_NAME"); + if (tableName.equalsIgnoreCase(TABLE)) { + needCreation = false; + break; + } + } + + if (needCreation) { + statement = connection.createStatement(); + statement.execute(DB_CREATE); + } + connection.commit(); + } catch (Exception e) { + errorOccurred = true; + throw new LogServiceException("Error setting up database", e); + } finally { + closeStatement(statement); + if (errorOccurred) { + closeConnection(connection); + } + } + + return connection; + } + + /** + * Inserts a StoredMsg as a row in the database table + * + * @param event + * @throws LogServiceException + */ + public synchronized void insert(StoredMsg event) throws LogServiceException { + boolean errorOccurred = false; + Connection conn = null; + + try { + conn = getConnection(); + if (saveStatement == null) { + saveStatement = conn + .prepareStatement(INSERT_PREPARED_STATEMENT); + } + + saveStatement.setTimestamp(1, new Timestamp(event.getEventTime() + .getTime())); + saveStatement.setString(2, event.getMessage()); + saveStatement.setString(3, event.getJavaClass()); + saveStatement.setInt(4, event.getLineNumber()); + saveStatement.setString(5, event.getStacktrace()); + saveStatement.setString(6, event.getThreadName()); + saveStatement.setString(7, event.getMachineName()); + saveStatement.setString(8, event.getLevel()); + int result = saveStatement.executeUpdate(); + if (result < 1) { + errorOccurred = true; + LogService.getLogger().error("Insert of logging event failed"); + } else { + conn.commit(); + } + } catch (Exception e) { + errorOccurred = true; + throw new LogServiceException( + "Error inserting event into log database", e); + } finally { + if (errorOccurred) { + closeStatement(saveStatement); + saveStatement = null; + closeConnection(conn); + } + } + } + + /** + * Gets the cached connection to the database + * + * @return + * @throws LogServiceException + */ + private synchronized Connection getConnection() throws LogServiceException { + if (connection != null) { + try { + if (connection.isClosed()) { + closeConnection(connection); + } + } catch (SQLException e) { + throw new LogServiceException("Error closing connection", e); + } + } + if (connection == null) { + try { + connection = createConnection(); + } catch (Exception e) { + throw new LogServiceException("Error creating connection", e); + } + } + + return connection; + } + + /** + * Closes a statement + * + * @param statement + */ + private void closeStatement(Statement statement) { + if (statement != null) { + try { + statement.close(); + } catch (SQLException e) { + // ignore + } + } + } + + /** + * Closes the connection to the database + * + * @param connection + */ + private void closeConnection(Connection connection) { + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + // ignore + } + connection = null; + } + + try { + DriverManager.getConnection("jdbc:derby:directory:" + + config.getDatabaseDir() + ";shutdown=true"); + } catch (SQLException e) { + // ignore as stop database will always throw an exception + } + } + + /** + * Closes a result set + * + * @param rs + */ + private void closeResultSet(ResultSet rs) { + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + // ignore + } + } + } + + /** + * Queries for rows that include stacktraces and correlates them into a + * LogReportContainer + * + * @return + * @throws LogServiceException + */ + private LogReportContainer queryEventsWithStacks() + throws LogServiceException { + Connection conn = null; + boolean errorOccurred = false; + Statement statement = null; + ResultSet rs = null; + LogReportContainer container = new LogReportContainer(); + try { + conn = getConnection(); + statement = conn.createStatement(); + // line number of 0 indicates that there was no stacktrace with + // the error, those will be handled separately + rs = statement + .executeQuery("select javaClass, lineNumber, machineName, count(*) " + + "from log where lineNumber != 0 " + + "group by javaClass, lineNumber, machineName"); + conn.commit(); + while (rs.next()) { + String javaClass = rs.getString(1); + int lineNumber = rs.getInt(2); + String machineName = rs.getString(3); + int count = rs.getInt(4); + LogReportEvent occ = new LogReportEvent(); + // errors sent in without a stacktrace will not have a line + // number of java class + occ.setJavaClass(javaClass); + occ.setLineNumber(lineNumber); + container.addError(occ, machineName, count); + } + return container; + } catch (SQLException e) { + errorOccurred = true; + throw new LogServiceException("Error executing query", e); + } finally { + closeResultSet(rs); + closeStatement(statement); + if (errorOccurred) { + closeConnection(conn); + } + } + } + + /** + * Loops through the LogReportContainer's events and queries for sample + * stacktraces and messages for them. + * + * @param container + * @throws LogServiceException + */ + private void addSamples(LogReportContainer container) + throws LogServiceException { + for (LogReportEvent event : container.getEvents()) { + addSamples(event); + } + } + + /** + * Fills in a LogReportEvent with a sample stacktrace and message + * + * @param event + * @throws LogServiceException + */ + private void addSamples(LogReportEvent event) throws LogServiceException { + Connection conn = null; + boolean errorOccurred = false; + Statement statement = null; + ResultSet rs = null; + try { + conn = getConnection(); + statement = conn.createStatement(); + statement.setFetchSize(1); + statement.setMaxRows(1); + rs = statement.executeQuery("select message, stacktrace, level " + + "from log where javaClass ='" + event.getJavaClass() + + "' and lineNumber = " + event.getLineNumber()); + conn.commit(); + while (rs.next()) { + String message = rs.getString(1); + String stacktrace = rs.getString(2); + String level = rs.getString(3); + event.setSampleMessage(message); + event.setSampleStacktrace(stacktrace); + event.setLevel(level); + } + } catch (SQLException e) { + errorOccurred = true; + throw new LogServiceException("Error executing query", e); + } finally { + closeResultSet(rs); + closeStatement(statement); + if (errorOccurred) { + closeConnection(conn); + } + } + } + + /** + * Queries for rows that don't have stacktraces and adds them to the + * LogReportContainer + * + * @param container + * @throws LogServiceException + */ + private void addNoStacktraceMessages(LogReportContainer container) + throws LogServiceException { + Connection conn = null; + boolean errorOccurred = false; + Statement statement = null; + ResultSet rs = null; + try { + conn = getConnection(); + statement = conn.createStatement(); + rs = statement + .executeQuery("select message, machineName, level, threadName, count(*) from log " + + "where lineNumber = 0 group by message, machineName, level, threadName"); + conn.commit(); + while (rs.next()) { + String message = rs.getString(1); + String machineName = rs.getString(2); + String level = rs.getString(3); + String thread = rs.getString(4); + int count = rs.getInt(5); + LogReportEvent event = new LogReportEvent(); + event.setSampleMessage(message); + event.setLevel(level); + event.setThreadName(thread); + container.addError(event, machineName, count); + } + } catch (SQLException e) { + errorOccurred = true; + throw new LogServiceException("Error executing query", e); + } finally { + closeResultSet(rs); + closeStatement(statement); + if (errorOccurred) { + closeConnection(conn); + } + } + } + + /** + * Queries for the earliest and latest time in the database and sets them on + * the LogReportContainer + * + * @param container + * @throws LogServiceException + */ + private void addTimes(LogReportContainer container) + throws LogServiceException { + boolean errorOccurred = false; + Connection conn = null; + ResultSet rs = null; + try { + conn = getConnection(); + if (timeStatement == null) { + timeStatement = conn.prepareStatement(TIME_STATEMENT); + } + rs = timeStatement.executeQuery(); + conn.commit(); + rs.next(); + container.setEarliestTime(rs.getTimestamp(1)); + container.setLatestTime(rs.getTimestamp(2)); + } catch (Exception e) { + errorOccurred = true; + throw new LogServiceException( + "Error determining min and max times", e); + } finally { + closeResultSet(rs); + if (errorOccurred) { + closeStatement(timeStatement); + timeStatement = null; + closeConnection(conn); + } + } + } + + /** + * Builds a LogReportContainer that correlates different rows as the same + * event, fills in sample stacktraces and messages, and includes the + * earliest and latest times of the events. + * + * @return + * @throws LogServiceException + */ + public LogReportContainer buildReport() throws LogServiceException { + LogReportContainer container = queryEventsWithStacks(); + addSamples(container); + addNoStacktraceMessages(container); + addTimes(container); + return container; + } + + /** + * Clears all rows from the database + * + * @throws LogServiceException + */ + public void clearEntries() throws LogServiceException { + boolean errorOccurred = false; + Connection conn = null; + + try { + conn = getConnection(); + if (purgeStatement == null) { + purgeStatement = conn.prepareStatement(PURGE_STATEMENT); + } + purgeStatement.executeUpdate(); + conn.commit(); + } catch (Exception e) { + errorOccurred = true; + throw new LogServiceException("Error purging message database", e); + } finally { + if (errorOccurred) { + closeStatement(purgeStatement); + purgeStatement = null; + closeConnection(conn); + } + } + } + + /** + * Queries the database to determine the number of rows + * + * @return + * @throws LogServiceException + */ + public int getCurrentRowCount() throws LogServiceException { + boolean errorOccurred = false; + Connection conn = null; + try { + conn = getConnection(); + if (countStatement == null) { + countStatement = conn.prepareStatement(COUNT_STATEMENT); + } + ResultSet rs = countStatement.executeQuery(); + conn.commit(); + rs.next(); + return rs.getInt(1); + } catch (Exception e) { + errorOccurred = true; + throw new LogServiceException("Error determining row count", e); + } finally { + if (errorOccurred) { + closeStatement(countStatement); + countStatement = null; + closeConnection(conn); + } + } + } + +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/quartz/CountRowsJob.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/quartz/CountRowsJob.java new file mode 100644 index 0000000000..ab56aea202 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/quartz/CountRowsJob.java @@ -0,0 +1,69 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv.quartz; + +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +import com.raytheon.uf.logsrv.LogService; +import com.raytheon.uf.logsrv.LogServiceException; +import com.raytheon.uf.logsrv.derby.DerbyDao; + +/** + * A simple quartz job that logs how many log events are currently in the + * database. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Sep 5, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class CountRowsJob implements Job { + + /* + * (non-Javadoc) + * + * @see org.quartz.Job#execute(org.quartz.JobExecutionContext) + */ + @Override + public void execute(JobExecutionContext arg0) throws JobExecutionException { + try { + int rowCount = DerbyDao.getInstance().getCurrentRowCount(); + LogService.getLogger() + .info("Database currently has " + rowCount + + " messages reported"); + } catch (LogServiceException e) { + LogService.getLogger().error( + "Error determining database row count", e); + } + + } + +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/quartz/CreateSendReportJob.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/quartz/CreateSendReportJob.java new file mode 100644 index 0000000000..0882400b02 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/quartz/CreateSendReportJob.java @@ -0,0 +1,103 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv.quartz; + +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +import com.raytheon.uf.logsrv.LogService; +import com.raytheon.uf.logsrv.LogServiceException; +import com.raytheon.uf.logsrv.config.LogSrvConfig; +import com.raytheon.uf.logsrv.derby.DerbyDao; +import com.raytheon.uf.logsrv.report.data.LogReportContainer; +import com.raytheon.uf.logsrv.report.email.HtmlGenerator; +import com.raytheon.uf.logsrv.report.email.ReportEmailer; + +/** + * A quartz job that queries the database to build report data, transforms the + * report data into HTML, emails the HTML to the configured address, and then + * purges the database of all logging events that were included in the report. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 30, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class CreateSendReportJob implements Job { + + /* + * (non-Javadoc) + * + * @see org.quartz.Job#execute(org.quartz.JobExecutionContext) + */ + @Override + public void execute(JobExecutionContext ctx) throws JobExecutionException { + LogService.getLogger().info( + "Create report job triggered, preparing to send report"); + DerbyDao dao = DerbyDao.getInstance(); + // synchronize on dao to prevent new log entries from being added + // while we're querying and then purging + synchronized (dao) { + LogReportContainer container = null; + try { + container = dao.buildReport(); + } catch (LogServiceException e) { + LogService.getLogger().error("Error building report", e); + throw new JobExecutionException("Error building report", e); + } + + String html = HtmlGenerator.generateHtml(container); + LogSrvConfig config = (LogSrvConfig) ctx.getJobDetail() + .getJobDataMap().get("config"); + + try { + ReportEmailer.email(html, config); + } catch (Exception e) { + LogService.getLogger().error("Error emailing report", e); + throw new JobExecutionException("Error emailing report", e); + } + LogService.getLogger().info( + "Report has been sent to: " + config.getToAddress()); + + // only clear out the database if the analysis report was + // successfully emailed + LogService.getLogger().info( + "Purging database of messages that were just reported"); + try { + dao.clearEntries(); + } catch (LogServiceException e) { + LogService.getLogger().error("Error purging database", e); + throw new JobExecutionException("Error purging database", e); + } + LogService.getLogger().info("Database purging complete"); + } + } + +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/quartz/JobScheduler.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/quartz/JobScheduler.java new file mode 100644 index 0000000000..eada4ad713 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/quartz/JobScheduler.java @@ -0,0 +1,75 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv.quartz; + +import org.quartz.JobDetail; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.SchedulerFactory; +import org.quartz.Trigger; +import org.quartz.TriggerUtils; +import org.quartz.impl.StdSchedulerFactory; + +import com.raytheon.uf.logsrv.config.LogSrvConfig; + +/** + * Schedules the quartz jobs in the main log process so at timed intervals, the + * number of logging events in the db are reported and the error report is + * generated and emailed. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 30, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class JobScheduler { + + public static void scheduleJobs(LogSrvConfig config) + throws SchedulerException { + SchedulerFactory factory = new StdSchedulerFactory(); + Scheduler sched = factory.getScheduler(); + sched.start(); + + JobDetail job = new JobDetail("Create and Send Report Job", + CreateSendReportJob.class); + job.getJobDataMap().put("config", config); + String[] split = config.getTimeToSend().split(":"); + int hour = Integer.parseInt(split[0]); + int minute = Integer.parseInt(split[1]); + Trigger trigger = TriggerUtils.makeDailyTrigger(hour, minute); + trigger.setName("Report Trigger"); + sched.scheduleJob(job, trigger); + + JobDetail countJob = new JobDetail("Count Rows Job", CountRowsJob.class); + Trigger countTrigger = TriggerUtils.makeHourlyTrigger(); + countTrigger.setName("Count Trigger"); + sched.scheduleJob(countJob, countTrigger); + } + +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/TestReportOutputter.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/TestReportOutputter.java new file mode 100644 index 0000000000..e954e8ec76 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/TestReportOutputter.java @@ -0,0 +1,67 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv.report; + +import java.io.File; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Unmarshaller; + +import com.raytheon.uf.logsrv.config.LogSrvConfig; +import com.raytheon.uf.logsrv.derby.DerbyDao; +import com.raytheon.uf.logsrv.report.data.LogReportContainer; +import com.raytheon.uf.logsrv.report.email.HtmlGenerator; +import com.raytheon.uf.logsrv.report.email.ReportEmailer; + +/** + * A simple main that generates a report based on the current entries in the + * database, emails the report, and then clears the database. Allows skipping + * waiting for the once a day email. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 28, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class TestReportOutputter { + + public static void main(String[] args) throws Exception { + JAXBContext context = JAXBContext.newInstance(LogSrvConfig.class); + Unmarshaller m = context.createUnmarshaller(); + LogSrvConfig config = (LogSrvConfig) m + .unmarshal(new File("config.xml")); + config.validate(); + DerbyDao.setConfig(config); + LogReportContainer container = DerbyDao.getInstance().buildReport(); + String report = HtmlGenerator.generateHtml(container); + ReportEmailer.email(report, config); + DerbyDao.getInstance().clearEntries(); + } + +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/data/LogReportContainer.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/data/LogReportContainer.java new file mode 100644 index 0000000000..1ea68b8096 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/data/LogReportContainer.java @@ -0,0 +1,90 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv.report.data; + +import java.sql.Timestamp; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * A container that holds a concept of a report based on the frequency of + * occurrences of logging events (ie errors) in the database. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 28, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class LogReportContainer { + + private Map map = new HashMap(); + + private Timestamp earliestTime; + + private Timestamp latestTime; + + public void addError(LogReportEvent event, String machineName, + int occurrences) { + String key = event.getKey(); + if (!map.containsKey(key)) { + map.put(key, event); + } + + LogReportEvent current = map.get(key); + Map machineCount = current.getMachineCount(); + Integer count = machineCount.get(machineName); + if (count == null) { + count = 0; + } + count += occurrences; + machineCount.put(machineName, count); + } + + public Collection getEvents() { + return map.values(); + } + + public Timestamp getEarliestTime() { + return earliestTime; + } + + public void setEarliestTime(Timestamp earliestTime) { + this.earliestTime = earliestTime; + } + + public Timestamp getLatestTime() { + return latestTime; + } + + public void setLatestTime(Timestamp latestTime) { + this.latestTime = latestTime; + } + +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/data/LogReportEvent.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/data/LogReportEvent.java new file mode 100644 index 0000000000..0ba60ef82f --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/data/LogReportEvent.java @@ -0,0 +1,184 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv.report.data; + +import java.util.HashMap; +import java.util.Map; + +import ch.qos.logback.classic.Level; + +/** + * An event representing a unique error that was originally reported as a + * logging event, with a count of occurrences per machine. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 28, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class LogReportEvent implements Comparable { + + private int lineNumber; + + private String javaClass; + + private String sampleStacktrace; + + private Map machineCount = new HashMap(); + + private String sampleMessage; + + private String level; + + private String threadName; + + public int getLineNumber() { + return lineNumber; + } + + public void setLineNumber(int lineNumber) { + this.lineNumber = lineNumber; + } + + public String getJavaClass() { + return javaClass; + } + + public void setJavaClass(String javaClass) { + this.javaClass = javaClass; + } + + public String getSampleStacktrace() { + return sampleStacktrace; + } + + public void setSampleStacktrace(String sampleStacktrace) { + this.sampleStacktrace = sampleStacktrace; + } + + public Map getMachineCount() { + return machineCount; + } + + public void setMachineCount(Map machineCount) { + this.machineCount = machineCount; + } + + public String getSampleMessage() { + return sampleMessage; + } + + public void setSampleMessage(String sampleMessage) { + this.sampleMessage = sampleMessage; + } + + public String getLevel() { + return level; + } + + public void setLevel(String level) { + this.level = level; + } + + public boolean receivedStack() { + return lineNumber > 0; + } + + public String getKey() { + String key = null; + if (receivedStack()) { + key = javaClass + ":" + lineNumber; + } else { + key = sampleMessage; + if (key == null) { + key = "null"; + } else if (key.length() > 20) { + // cut off the key to try and match it to others + key = key.substring(0, 20); + } + + } + return key; + } + + public int getTotalOccurences() { + int count = 0; + for (Integer i : machineCount.values()) { + count += i; + } + return count; + } + + /** + * Compares the LogReportEvents to enable sorting. Sort order goes as + * follows: warn, no stacktraces, and least number of occurrences error, no + * stacktraces, and least number of occurrences stacktrace, warn, and least + * number of occurrences stacktrace, error, and least number of occurrences + * + * For example, the highest/last in the sort would be the ERROR with a + * stacktrace that occurred the most. + */ + @Override + public int compareTo(LogReportEvent o) { + int retVal = 0; + if (!this.receivedStack() && o.receivedStack()) { + retVal = -1; + } else if (this.receivedStack() && !o.receivedStack()) { + retVal = 1; + } else { + // both match on having a stacktrace or not, rank them + // based on error or warning + Level thisLevel = Level.valueOf(level); + Level oLevel = Level.valueOf(o.getLevel()); + if (thisLevel.toInt() < oLevel.toInt()) { + retVal = -1; + } else if (thisLevel.toInt() > oLevel.toInt()) { + retVal = 1; + } else { + // both match on having a stacktrace or not, and match + // on being WARNs or ERRORs, so rank them on number + // of times they occurred + int count1 = this.getTotalOccurences(); + int count2 = o.getTotalOccurences(); + retVal = (count1 < count2) ? -1 : ((count1 == count2) ? 0 : 1); + } + } + + return retVal; + } + + public String getThreadName() { + return threadName; + } + + public void setThreadName(String threadName) { + this.threadName = threadName; + } + +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/email/HtmlGenerator.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/email/HtmlGenerator.java new file mode 100644 index 0000000000..fa476f6485 --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/email/HtmlGenerator.java @@ -0,0 +1,194 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv.report.email; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import com.raytheon.uf.logsrv.report.data.LogReportContainer; +import com.raytheon.uf.logsrv.report.data.LogReportEvent; + +/** + * Uses a LogReportContainer to generate an HTML report that can be emailed. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 28, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class HtmlGenerator { + + private static final int MAX_ERRORS = 100; + + private static final String LINE_BREAK = "
"; + + private static final String HR = "
"; + + private static final SimpleDateFormat SDF = new SimpleDateFormat( + "yyyy-MM-dd HH:mm z"); + + /** + * Generates an HTML string based on the report container + * + * @param container + * the container of events to generate HTML for + * @return an HTML formatted report + */ + public static String generateHtml(LogReportContainer container) { + String report = null; + if (container != null) { + Collection collection = container.getEvents(); + LogReportEvent[] array = collection.toArray(new LogReportEvent[0]); + Arrays.sort(array, Collections.reverseOrder()); + + StringBuilder sb = new StringBuilder(); + sb.append(buildHeader(array.length, container.getEarliestTime(), + container.getLatestTime())); + sb.append(LINE_BREAK); + boolean foundFirstWithoutStacktrace = false; + for (int i = 0; i < array.length && i < MAX_ERRORS; i++) { + LogReportEvent event = array[i]; + if (!event.receivedStack() && !foundFirstWithoutStacktrace) { + foundFirstWithoutStacktrace = true; + sb.append(buildNoStacktraceDisclaimer()); + } + sb.append(HR); + sb.append(LINE_BREAK); + sb.append(buildEvent(array[i])); + sb.append(LINE_BREAK); + sb.append(LINE_BREAK); + } + report = sb.toString(); + } + + return report; + } + + private static CharSequence buildHeader(int distinctErrors, Date earliest, + Date latest) { + StringBuilder sb = new StringBuilder(); + sb.append("Auto-generated error report"); + sb.append(LINE_BREAK); + sb.append("Created on "); + sb.append(SDF.format(new Date())); + sb.append(LINE_BREAK); + if (distinctErrors > 0) { + sb.append("Timeframe of errors: "); + sb.append(SDF.format(earliest)); + sb.append(""); + sb.append(" to "); + sb.append(SDF.format(latest)); + sb.append(""); + sb.append(LINE_BREAK); + } + sb.append("Number of distinct errors reported: "); + sb.append(distinctErrors); + sb.append(""); + sb.append(LINE_BREAK); + if (distinctErrors > MAX_ERRORS) { + sb.append("Report truncated to top "); + sb.append(MAX_ERRORS); + sb.append(" errors/warnings"); + sb.append(LINE_BREAK); + } + + return sb; + } + + private static CharSequence buildEvent(LogReportEvent event) { + StringBuilder sb = new StringBuilder(); + sb.append(""); + sb.append(event.getLevel()); + if (!event.receivedStack()) { + // thread name is really only useful if we don't have a stacktrace + sb.append(" ["); + sb.append(event.getThreadName()); + sb.append("]"); + } + sb.append(" "); + sb.append(event.getSampleMessage()); + sb.append(""); + sb.append(LINE_BREAK); + sb.append("Occurred a total of "); + sb.append(event.getTotalOccurences()); + sb.append(" times on the following machines:"); + sb.append(LINE_BREAK); + Map machineCount = event.getMachineCount(); + Set> set = machineCount.entrySet(); + List> list = new ArrayList>( + set); + // sort it so the machines with the most errors are listed first + Collections.sort(list, new Comparator>() { + @Override + public int compare(Entry o1, + Entry o2) { + int val1 = o1.getValue(); + int val2 = o2.getValue(); + return (val2 < val1) ? -1 : ((val1 == val2) ? 0 : 1); + } + + }); + + for (Entry entry : list) { + sb.append(""); + sb.append(entry.getKey()); + sb.append(": "); + sb.append(entry.getValue()); + sb.append(" occurrences"); + sb.append(LINE_BREAK); + } + sb.append(LINE_BREAK); + if (event.receivedStack()) { + sb.append(event.getSampleStacktrace().replaceAll("\n", LINE_BREAK)); + } + sb.append(LINE_BREAK); + return sb; + } + + private static CharSequence buildNoStacktraceDisclaimer() { + StringBuilder sb = new StringBuilder(); + sb.append(LINE_BREAK); + sb.append(HR); + sb.append("Disclaimer: The following errors were received without stacktraces and..."); + sb.append("
  • Are estimated/inexact
    • "); + sb.append("
  • Should be considered for downgrade to INFO or DEBUG messages"); + sb.append("
  • Should include stacktraces if remaining as ERROR
"); + return sb; + } + +} diff --git a/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/email/ReportEmailer.java b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/email/ReportEmailer.java new file mode 100644 index 0000000000..31befab29c --- /dev/null +++ b/javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/report/email/ReportEmailer.java @@ -0,0 +1,83 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.logsrv.report.email; + +import java.util.Properties; + +import javax.mail.Message; +import javax.mail.Session; +import javax.mail.Transport; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeMessage; + +import com.raytheon.uf.logsrv.config.LogSrvConfig; + +/** + * Emails a report using the options specified in the config. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 29, 2013            njensen     Initial creation
+ * 
+ *
+ * + * @author njensen + * @version 1.0 + */ + +public class ReportEmailer { + + /** + * Emails the provided string, using the options from the config. + * + * @param report + * the text to email + * @param config + * the config of email options + * @throws Exception + */ + public static void email(String report, LogSrvConfig config) + throws Exception { + Properties props = new Properties(); + props.put("host", config.getSmtpHost()); + props.put("mail.smtp.user", config.getFromAddress()); + props.put("port", config.getSmtpPort()); + + Session session = Session.getDefaultInstance(props, null); + MimeMessage message = new MimeMessage(session); + message.setFrom(new InternetAddress(config.getFromAddress())); + String[] split = config.getToAddress().split(","); + for (String to : split) { + message.addRecipient(Message.RecipientType.TO, new InternetAddress( + to.trim())); + } + message.setSubject(config.getClusterName() + " error report"); + message.setContent(report, "text/html; charset=utf-8"); + Transport transport = session.getTransport("smtp"); + transport.connect(config.getSmtpHost(), config.getFromAddress(), null); + transport.sendMessage(message, message.getAllRecipients()); + transport.close(); + } + +}