Issue #3470 - the wrapperCapture script is now packaged in the awips2-edex and awips2-qpid-java-broker rpms. additional

log4j jar is now included in the awips2-qpid-java-common rpm. Fixed AbstractScript import error.

Change-Id: I7acfdff45944b8d9dd2f75e85cca3cc6d54b1903

Former-commit-id: 632e493332 [formerly 09442217cd] [formerly a9b4b4124a] [formerly b568c25e10 [formerly a9b4b4124a [formerly d11649deb78aa356f27a22f33b38f7238513576c]]]
Former-commit-id: b568c25e10
Former-commit-id: 74192b218460ef7622e9a89c78967d022d36f146 [formerly 1d6eb95f9f]
Former-commit-id: b0c94a2d61

edexOsgi/build.edex/esb/bin/yajsw/scripts/wrapperCapture.sh

@@ -0,0 +1,180 @@
+#!/bin/sh
+#####################################################################
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+#####################################################################
+#####################################################################
+# Script for capturing data from a wrapper java process when the
+# wrapper restarts the process
+#
+# SOFTWARE HISTORY
+#
+# Date          Ticket#  Engineer    Description
+# ------------- -------- ----------- --------------------------
+# Aug 07, 2014  3470     rjpeter     Initial creation
+#
+#####################################################################
+# NOTE: Script must be located at /awips2/qpid/bin/yajsw/scripts for it to work
+
+# base path to save capture data to, will create subdirectory for each server
+basePath="/data/fxa/cave"
+
+state=$1
+string_state=$2
+pid=$4
+
+path_to_script=`readlink -f $0`
+curTime=`date +%Y%m%d_%H%M%S`
+echo "$curTime: Wrapper running $path_to_script due to state transition for pid $pid.  New State $state|$string_state"
+
+# ensure directory is created and has write permissions
+checkDir() {
+   dir="$1"
+   if [ ! -d "$dir" ]; then
+      mkdir -p $dir
+      if [ ! -d "$dir" ]; then
+         message="Unable to create qpid capture data directory\n$dir"
+         echo -e "Capture failed: $message"
+         exit 1
+      fi
+   fi
+
+   if [ ! -w "$dir" ]; then
+      message="Do not have write permissions to qpid capture data directory\n$dir"
+      echo -e "Capture failed: $message"
+      exit 1
+   fi
+}
+
+# gets top output of local server
+runTop() {
+   local curTime=`date "+%Y%m%d_%H:%M:%S"`
+   echo "$curTime: Capturing top"
+   echo "$curTime: Capturing top" >> $processFile
+   local out_file="${dataPath}/top.log"
+   export COLUMNS=160
+   top -b -c -n1 >> $out_file 2>&1
+   curTime=`date +%Y%m%d_%H%M%S`
+   echo "$curTime: top captured"
+}
+
+# runs jstack 10 times, if it fails will run again with -F
+runJstack() {
+   local curTime=`date +%Y%m%d_%H%M%S`
+   echo "$curTime: Capturing jstacks"
+   local pid="$1"
+   local count=1
+   local cmd="/awips2/java/bin/jstack"
+   local prePath="${dataPath}/pid_${pid}_"
+   local log=""
+
+   while [ "$count" -le "10" ]; do
+      curTime=`date "+%Y%m%d_%H:%M:%S"`
+      log="${prePath}jstack_${count}.log"
+
+      echo "${curTime}: Running command: ${cmd} ${pid} >> ${log} 2>&1" >> $processFile
+      echo "Running for $curTime" >> $log
+      ${cmd} ${pid} >> ${log} 2>&1
+
+      if [[ "$?" != "0" && $FORCE != "y" ]]; then
+         curTime=`date "+%Y%m%d_%H:%M:%S"`
+         echo "${curTime}: jstack for $pid failed to connect, rerunning with -F" >> $processFile
+         ${cmd} -F ${pid} >> ${log} 2>&1
+      fi
+      let "count+=1"
+   done
+
+   curTime=`date +%Y%m%d_%H%M%S`
+   echo "$curTime: jstacks captured"
+}
+
+# runs jmap -heap
+runJmapHeap() {
+   local curTime=`date +%Y%m%d_%H%M%S`
+   echo "$curTime: Capturing jmap -heap"
+   local pid=$1
+   local prePath="${dataPath}/pid_${pid}_"
+
+   local log="${prePath}jmapHeap.log"
+   local cmd="/awips2/java/bin/jmap -heap"
+   echo "${curTime}: Running command: $cmd $pid >> $log 2>&1" >> $processFile
+   $cmd $pid >> $log 2>&1
+
+   if [[ "$?" != "0" && $FORCE != "y" ]]; then
+      curTime=`date "+%Y%m%d_%H:%M:%S"`
+      echo "${curTime}: jmap for $pid failed to connect, rerunning with -F" >> $processFile
+      $cmd -F $pid >> $log 2>&1
+   fi
+
+   curTime=`date +%Y%m%d_%H%M%S`
+   echo "$curTime: jmap -heap captured"
+}
+
+# runs jmap, if it fails will run again with -F
+runJmap() {
+   local curTime=`date +%Y%m%d_%H%M%S`
+   echo "$curTime: Capturing jmap -dump"
+   local pid=$1
+   local prePath="${dataPath}/pid_${pid}_jmap"
+
+   local log="${prePath}.log"
+   local dumpPath="${prePath}.hprof"
+   local cmd="/awips2/java/bin/jmap -dump:format=b,file=${dumpPath}"
+   echo "${curTime}: Running command: $cmd $pid >> $log 2>&1" >> $processFile
+   $cmd $pid >> $log 2>&1
+
+   if [[ "$?" != "0" && $FORCE != "y" ]]; then
+      curTime=`date "+%Y%m%d_%H:%M:%S"`
+      echo "${curTime}: jmap for $pid failed to connect, rerunning with -F" >> $processFile
+      $cmd -F $pid >> $log 2>&1
+   fi
+
+   curTime=`date +%Y%m%d_%H%M%S`
+   echo "$curTime: jmap -dump captured"
+}
+
+
+
+if [[ "$pid" != "-1" ]]; then
+   process=`ps -ef | grep $pid | grep java`
+
+   if [[ "$process" != "" ]]; then
+   hostName=`hostname -s`
+   dataPath="${basePath}/${hostName}/wrapperCaptureData_${curTime}_pid_$pid"
+   checkDir $dataPath
+   processFile=${dataPath}/capture_info.log
+   echo "Wrapper running $0 due to state transition for pid $pid.  New State $state|$string_state" >> $processFile
+   echo "Process information:" >> $processFile
+   ps -ef | grep $pid >> $processFile
+   runTop &
+   runJstack $pid &
+   runJmapHeap $pid &
+   # TODO: Double check if jvm already dumped one
+   runJmap $pid &
+   wait
+
+   curTime=`date +%Y%m%d_%H%M%S`
+   echo "$curTime: Data captured to $dataPath"
+   else
+      curTime=`date +%Y%m%d_%H%M%S`
+      echo "$curTime: PID $pid is no longer running, nothing to capture"
+   fi
+else
+   curTime=`date +%Y%m%d_%H%M%S`
+   echo "$curTime: PID was -1, process no longer running, nothing to capture"
+fi

javaUtilities/yajsw/deploy.xml

@@ -3,6 +3,10 @@
 		determine where the various destination directories are
 		without ant-contrib
 	-->
+	
+	<!-- the location of javaUtilities -->
+	<dirname property="jutilities.dir" file="${basedir}" />
+	
 	<!-- the EDEX destination -->
 	<available file="${basedir}/../build.edex"
 		property="edex.destination"
@@ -73,6 +77,14 @@
 				<exclude name="**/ReadMe.txt" />
 			</fileset>
 		</copy>
+		<!-- copy scripts -->
+		<mkdir dir="${edex.bin.directory}/yajsw/scripts" />
+		<copy todir="${edex.bin.directory}/yajsw/scripts"
+		failonerror="true" verbose="true" overwrite="true">
+			<fileset dir="${jutilities.dir}/yajsw-scripts">
+				<include name="*.sh" />
+			</fileset>
+		</copy>
 		<!-- deploy to QPID -->
 		<tar destfile="${qpid.destination}/SOURCES/yajsw-distribution.tar"
 			basedir="${edex.bin.directory}"

javaUtilities/yajsw/src/main/java/org/rzo/yajsw/script/AbstractScript.java

@@ -16,6 +16,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import org.jboss.netty.util.HashedWheelTimer;
 import org.jboss.netty.util.Timeout;
@@ -28,29 +29,27 @@ import org.rzo.yajsw.wrapper.WrappedProcess;
 /**
  * The Class AbstractScript.
  */
-public abstract class AbstractScript implements Script
-{
+public abstract class AbstractScript implements Script {
 
 	/** The _name. */
-	String					_name;
+	String _name;
 
 	/** The _timeout. */
-	int						_timeout	= 30000;
+	int _timeout = 30000;
 
-	WrappedProcess			_process;
+	WrappedProcess _process;
 
-	String					_id;
+	String _id;
+
+	String[] _args;
 
-	String[]				_args;
-	
 	final static Timer TIMER = new HashedWheelTimer();
-	static final ExecutorService	EXECUTOR		= (ThreadPoolExecutor) new ThreadPoolExecutor(0, 50, 120L, TimeUnit.SECONDS,
-			new SynchronousQueue<Runnable>(), new DaemonThreadFactory("scriptExecutorInternal"));
+	static final ExecutorService EXECUTOR = (ThreadPoolExecutor) new ThreadPoolExecutor(
+			0, 50, 120L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+			new DaemonThreadFactory("scriptExecutorInternal"));
 	volatile Future _future;
 	volatile Timeout _timerTimeout;
 
-
-
 	/**
 	 * Instantiates a new abstract script.
 	 * 
@@ -58,8 +57,8 @@ public abstract class AbstractScript implements Script
 	 *            the script
 	 * @param timeout
 	 */
-	public AbstractScript(String script, String id, WrappedProcess process, String[] args, int timeout)
-	{
+	public AbstractScript(String script, String id, WrappedProcess process,
+			String[] args, int timeout) {
 		_name = script;
 		_process = process;
 		_id = id;
@@ -76,34 +75,35 @@ public abstract class AbstractScript implements Script
 	 * java.lang.String, java.lang.Object)
 	 */
 	public abstract Object execute(String line);
+
 	public abstract void interrupt();
+
 	abstract void log(String msg);
 
-	synchronized public void executeWithTimeout(final String line)
-	{
-        /**
-         * Changed by rjpeter  Aug 07, 2014.
-         */
-        _future = EXECUTOR.submit(new Callable<Object>() {
-            @Override
-            public Object call() {
-                return execute(line);
-            }
-        });
+	synchronized public void executeWithTimeout(final String line) {
+		/**
+		 * Changed by rjpeter Aug 07, 2014.
+		 */
+		_future = EXECUTOR.submit(new Callable<Object>() {
+			@Override
+			public Object call() {
+				return execute(line);
+			}
+		});
 
-        // wait for script to finish
-        try {
-            _future.get(_timeout, TimeUnit.MILLISECONDS);
-        } catch (TimeoutException e) {
-            log("script " + _name + " took too long -> interrupt");
-            try {
-                interrupt();
-            } catch (Throwable e1) {
+		// wait for script to finish
+		try {
+			_future.get(_timeout, TimeUnit.MILLISECONDS);
+		} catch (TimeoutException e) {
+			log("script " + _name + " took too long -> interrupt");
+			try {
+				interrupt();
+			} catch (Throwable e1) {
 
-            }
-        } catch (Exception e) {
+			}
+		} catch (Exception e) {
 
-        }
+		}
 	}
 
 	/*
@@ -111,8 +111,7 @@ public abstract class AbstractScript implements Script
 	 * 
 	 * @see org.rzo.yajsw.script.Script#getScript()
 	 */
-	public String getScript()
-	{
+	public String getScript() {
 		return _name;
 	}
 
@@ -121,8 +120,7 @@ public abstract class AbstractScript implements Script
 	 * 
 	 * @return the timeout
 	 */
-	public int getTimeout()
-	{
+	public int getTimeout() {
 		return _timeout;
 	}
 
@@ -132,8 +130,7 @@ public abstract class AbstractScript implements Script
 	 * @param timeout
 	 *            the new timeout
 	 */
-	public void setTimeout(int timeout)
-	{
+	public void setTimeout(int timeout) {
 		_timeout = timeout;
 	}