Merge "Issue #2359 clean up log service for distribution, build everything necessary into tar.gz file" into omaha_14.2.1

Former-commit-id: fc700573f5 [formerly 21343b9936] [formerly aa2f9a4fb9] [formerly cf2b175d9a [formerly aa2f9a4fb9 [formerly 27bfbf913f7dcf635d0c00100c05f0281ba37dae]]]
Former-commit-id: cf2b175d9a
Former-commit-id: 02dcf19bacd8f7b56dc58c64c9ceea8580ad84ce [formerly fef22a1cee]
Former-commit-id: 97a17db973

javaUtilities/com.raytheon.uf.logsrv/META-INF/MANIFEST.MF

@@ -1,9 +1,10 @@
 Manifest-Version: 1.0
+Main-Class: com.raytheon.uf.logsrv.LogService
+Class-Path: logback-core-1.0.13.jar logback-classic-1.0.13.jar derby.jar mail.jar slf4j-api-1.7.5.jar quartz-1.8.6.jar commons-lang-2.3.jar
 Bundle-ManifestVersion: 2
 Bundle-Name: Logsrv
 Bundle-SymbolicName: com.raytheon.uf.logsrv
-Bundle-Version: 1.0.0.qualifier
+Bundle-Version: 1.14.0.qualifier
-Bundle-Activator: com.raytheon.uf.logsrv.Activator
 Bundle-Vendor: RAYTHEON
 Bundle-RequiredExecutionEnvironment: JavaSE-1.6
 Bundle-ActivationPolicy: lazy

javaUtilities/com.raytheon.uf.logsrv/README.txt

@@ -0,0 +1,118 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+
+Build instructions
+-----------------------
+In Eclipse run the build.xml by right clicking on the build.xml file and
+running it.  That will produce a tar.gz with everything you need inside of it.
+
+
+Install instructions
+----------------------- 
+Place the tar.gz file where you would like to install it, and run tar -xvf on
+the tar.gz file.  Next go into the conf directory and modify config.xml to
+ the best settings for your cluster.  Then open receiver.xml and go to the
+ bottom of the file.  Edit the address to the name of the machine where
+ you are installing the log service, and pick a port if you are not happy with
+ the default.  Once your config settings are right, run the bin/logsrv.sh
+ script to start the log service.
+ 
+ At this point the log service is running but nothing is reporting to it.  You
+ can configure any Java process using logback to report to it, but with the
+ current version that is just EDEX and CAVE.  (Note technically you could
+ configure AlertViz to report to the service too, but that is fairly pointless).
+ 
+ To configure EDEX or CAVE to report to the log service, find the logback
+ config files for those applications.  In EDEX they can typically be found at
+ /awips2/edex/conf.  If you are not sure which logback file corresponds to
+ which JVM, look at the shell scripts in /awips2/edex/etc.  If not explicitly
+ stated in their corresponding shell script, then the JVMs use the logback file
+ specified in default.sh.
+ 
+ In CAVE the logback config files can typically be found at
+ /awips2/cave/plugins/com.raytheon.uf.viz.core_${VERSION}.
+ 
+ Once found, use a text editor to open the logback config file corresponding
+ to the process you wish to report to the log service.  You need to add two
+ things to the file, an appender and an appender-ref.
+ 
+ Add an appender like the example below in the appenders section (generally
+ towards the top of the file):
+ 
+<appender class="ch.qos.logback.classic.net.SocketAppender" name="remoteLogSrv">
+   <includeCallerData>false</includeCallerData>
+   <port>5477</port>
+   <remoteHost>dev33.oma.us.ray.com</remoteHost>
+   <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+   <level>WARN</level>
+   </filter>
+</appender>
+ 
+ Replace the remoteHost address with the machine where you installed the
+ log service.  Note that you must have a network route between the two machines
+ as it uses a socket to report the errors.  Replace the port with the port you chose
+ in receiver.xml.  You can alter the threshold if need be.
+ 
+ Next you must add your new remoteLogSrv appender to a logger.  Note that a
+ logger can have multiple appenders.  For EDEX, the recommendation is to add
+ it to the root logger.  For example:
+ 
+<root>
+   <level value="INFO"/>
+   <appender-ref ref="asyncConsole"/>
+   <appender-ref ref="remoteLogSrv"/>
+</root>
+ 
+ For CAVE, the recommendation is to add it to the CaveLogger.  For example:
+ 
+<logger name="CaveLogger" additivity="false">
+   <level value="ALL"/>
+   <appender-ref ref="AsyncCaveLogAppender"/>
+   <appender-ref ref="remoteLogSrv"/>
+</logger>
+
+Once you save the modified logback config file, you're done.  Logback will
+automatically pick up a changed configuration within a minute or two, and
+the Java processes you configured will start appending to the socket.
+
+
+How it works
+--------------
+The log service is listening on the socket and will store the messages in a
+derby database on the filesystem, and then at the scheduled time it will
+attempt to analyze and consolidate the errors based on the information it has
+available.  Then it will send an email using the configuration, reporting in an order
+of what errors it thinks are most significant.
+
+Note that it does not matter if the Java processes have the socket appender
+configured but the log service is not running.  They will try to connect but then
+go back to normal logging.  If the log service is running, they will resume sending
+log messages through the socket.  Therefore, it does not hurt to have
+everything configured even if the log service or the reporting processes are not
+running.
+
+
+Bugs and/or Improvements
+--------------
+If you encounter a bug with the log service or have an idea of how it can be
+improved, send an email to nathan.jensen@raytheon.com.
+

javaUtilities/com.raytheon.uf.logsrv/build.xml

@@ -0,0 +1,108 @@
+<!--
+    This_software_was_developed_and_/_or_modified_by_Raytheon_Company,
+    pursuant_to_Contract_DG133W-05-CQ-1067_with_the_US_Government.
+    
+    U.S._EXPORT_CONTROLLED_TECHNICAL_DATA
+    This_software_product_contains_export-restricted_data_whose
+    export/transfer/disclosure_is_restricted_by_U.S._law._Dissemination
+    to_non-U.S._persons_whether_in_the_United_States_or_abroad_requires
+    an_export_license_or_other_authorization.
+    
+    Contractor_Name:________Raytheon_Company
+    Contractor_Address:_____6825_Pine_Street,_Suite_340
+    ________________________Mail_Stop_B8
+    ________________________Omaha,_NE_68106
+    ________________________402.291.0100
+    
+    See_the_AWIPS_II_Master_Rights_File_("Master_Rights_File.pdf")_for
+    further_licensing_information.
+-->
+<project name="com.raytheon.uf.logsrv" default="dist">
+  
+  <property name="foss" location="../../cots/"/>
+  <property name="src" location="src"/>
+  <property name="bin" location="bin"/>
+  <property name="qualifier" value="${date}"/>
+  <property name="prefix" value="logsrv"/>
+  
+  <target name="init">
+    <mkdir dir="${bin}"/>
+    <tstamp>  
+      <format property="date" pattern="yyyyMMdd" />  
+    </tstamp>  
+  </target>
+	
+  <target name="compile" depends="init">
+    <javac srcdir="${src}" destdir="${bin}"/>
+  </target>
+	
+  <target name="copyManifest" depends="init">
+    <loadproperties srcfile="META-INF/MANIFEST.MF">
+      <filterchain>
+        <linecontainsregexp>
+          <regexp pattern="^Bundle-Version: \d+.\d+.\d+.qualifier$" />
+        </linecontainsregexp>
+        <tokenfilter>
+          <replaceregex pattern="qualifier$" replace="${qualifier}" />
+        </tokenfilter>
+      </filterchain>
+    </loadproperties>
+    <copy file="META-INF/MANIFEST.MF" tofile="${bin}/META-INF/MANIFEST.MF" />
+  	<manifest file="${bin}/META-INF/MANIFEST.MF" mode="update">
+      <attribute name="Bundle-Version" value="${Bundle-Version}" />
+    </manifest>
+  </target>
+
+  <target name="build" depends="compile,copyManifest">
+    <jar destfile="${ant.project.name}.jar" basedir="${bin}" manifest="${bin}/META-INF/MANIFEST.MF"/>
+  </target>
+
+  <target name="clean">
+    <delete dir="${bin}"/>
+    <delete verbose="true">
+      <fileset dir="." includes="${ant.project.name}-*.jar"/>
+    </delete>
+  </target>
+	
+	<target name="tar" depends="build">
+		<tar destfile="${ant.project.name}-${Bundle-Version}.tar.gz" compression="gzip">
+			<tarfileset dir="${foss}/org.slf4j" prefix="${prefix}/lib/">
+			   <include name="slf4j-api*.jar"/>
+				<exclude name="*sources.jar"/>
+			</tarfileset>
+			<tarfileset dir="${foss}/ch.qos.logback" prefix="${prefix}/lib/">
+			    <include name="logback*.jar"/>
+				<exclude name="*sources.jar"/>
+			</tarfileset>
+            <tarfileset dir="${foss}/org.apache.derby" prefix="${prefix}/lib/">
+               <include name="derby*.jar"/>
+            </tarfileset>
+            <tarfileset dir="${foss}/javax.mail" prefix="${prefix}/lib/">
+               <include name="mail.jar"/>
+            </tarfileset>
+            <tarfileset dir="${foss}/org.quartz" prefix="${prefix}/lib/">
+                   <include name="quartz*.jar"/>
+            </tarfileset>			
+            <tarfileset dir="${foss}/org.apache.commons.lang" prefix="${prefix}/lib/">
+               <include name="commons-lang*.jar"/>
+            </tarfileset>
+            <tarfileset dir="./" prefix="${prefix}/lib/">
+               <include name="${ant.project.name}.jar"/>
+            </tarfileset>
+			<tarfileset dir="conf" prefix="${prefix}/conf/">
+               <include name="*.xml"/>               		
+			</tarfileset>
+	         <tarfileset dir="./" prefix="${prefix}/bin/" filemode="754">
+	            <include name="*.sh"/>	                     
+	         </tarfileset>
+             <tarfileset dir="./" prefix="${prefix}/">
+                <include name="README.txt"/>
+             </tarfileset>			
+		</tar>
+	</target>		
+	
+	<target name="dist" depends="tar">
+		<delete file="${ant.project.name}.jar"/>
+	</target>		
+
+</project>

javaUtilities/com.raytheon.uf.logsrv/conf/config.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<logSrvConfig>
+
+  <!-- the cluster name, only used for the report email -->
+  <clusterName>ec-oma</clusterName>
+  
+  <!-- where to keep track of the errors
+             how much space you need depends on how many errors the system is throwing
+             -->
+  <databaseDir>/common/njensen/logsrv/</databaseDir>
+  
+  <!--  how to send the email report -->
+  <fromAddress>Nathan.Jensen@raytheon.com</fromAddress>
+  <smtpHost>mk2-msg10.raymail.ray.com</smtpHost>
+  <smtpPort>143</smtpPort>
+  
+  <!-- where to send the email report, a comma-separated list of addresses -->
+  <toAddress>awipsctl@list.app.ray.com, awipstest@list.app.ray.com, david_j_hladky@raytheon.com</toAddress>
+
+  <!-- the time of day to send the report
+              only really matters if you're installing or auto-deploying at a
+              specific time, as you may want to clear out the databaseDir
+              that contains errors from a previous build
+              -->  
+  <timeToSend>00:45</timeToSend>
+  
+  <!--  threads to ignore errors on to keep them from overpowering the other errors -->
+  <ignoreThreads>shefThreadPool</ignoreThreads>
+  
+</logSrvConfig>

javaUtilities/com.raytheon.uf.logsrv/conf/receiver.xml

@@ -5,8 +5,8 @@
   
   <appender name="InternalLog" class="ch.qos.logback.core.rolling.RollingFileAppender">
       <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
-         <fileNamePattern>/home/njensen/logs/logService-internal-%d{yyyyMMdd}.log</fileNamePattern>
+         <fileNamePattern>${logSrvLogs}/logs/logService-internal-%d{yyyyMMdd}.log</fileNamePattern>
-         <maxHistory>30</maxHistory>
+         <maxHistory>7</maxHistory>
       </rollingPolicy> 
       <encoder> 
          <pattern>%-5p %d [%t] %c{0}: %m%n</pattern> 

javaUtilities/com.raytheon.uf.logsrv/config.xml

@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<logSrvConfig>
-  <clusterName>ec-oma</clusterName>
-  <databaseDir>/awips2/edex/data/utility/nate</databaseDir>
-  <fromAddress>Nathan.Jensen@raytheon.com</fromAddress>
-  <smtpHost>mk2-msg10.raymail.ray.com</smtpHost>
-  <smtpPort>143</smtpPort>
-  <toAddress>awipsctl@list.app.ray.com, awipstest@list.app.ray.com, david_j_hladky@raytheon.com</toAddress>
-  <!-- >toAddress>Nathan.Jensen@raytheon.com</toAddress-->
-  <timeToSend>00:45</timeToSend>
-  <ignoreThreads>shefThreadPool</ignoreThreads>
-</logSrvConfig>

javaUtilities/com.raytheon.uf.logsrv/logsrv.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+# 
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+# 
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+# 
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+path_to_script=`readlink -f $0`
+binDir=$(dirname $path_to_script)
+logSrvDir=$(dirname $binDir)
+export logSrvConf=$logSrvDir/conf/
+export logSrvLogs=$logSrvDir/logs/
+
+java -jar ${logSrvDir}/lib/com.raytheon.uf.logsrv.jar

javaUtilities/com.raytheon.uf.logsrv/src/com/raytheon/uf/logsrv/LogService.java

@@ -58,6 +58,8 @@ public class LogService {
 
     private static final String SERVICE_CONFIG = "config.xml";
 
+    private static final String ENV_CONF_DIR = "logSrvConf";
+
     private static final Logger logger = LoggerFactory
             .getLogger("InternalLogger");
 
@@ -67,11 +69,17 @@ public class LogService {
      */
     public static void main(String[] args) throws Exception {
         logger.info("Starting log analytics service");
+        String confDir = System.getenv(ENV_CONF_DIR);
+        if (confDir == null) {
+            throw new LogServiceException("Environment variable "
+                    + ENV_CONF_DIR
+                    + " is not set! Unable to find configuration!");
+        }
 
         JAXBContext context = JAXBContext.newInstance(LogSrvConfig.class);
         Unmarshaller m = context.createUnmarshaller();
-        LogSrvConfig config = (LogSrvConfig) m.unmarshal(new File(
+        LogSrvConfig config = (LogSrvConfig) m.unmarshal(new File(confDir
-                SERVICE_CONFIG));
+                + SERVICE_CONFIG));
         config.validate();
         DerbyDao.getInstance().setConfig(config);
         logger.info("Logging events from " + config.getClusterName());
@@ -81,7 +89,7 @@ public class LogService {
         lc.reset();
         JoranConfigurator configurator = new JoranConfigurator();
         configurator.setContext(lc);
-        configurator.doConfigure(LOGBACK_CONFIG);
+        configurator.doConfigure(confDir + LOGBACK_CONFIG);
 
         logger.info("Scheduling report generation");
         JobScheduler.scheduleJobs(config);