From f23007e32bdd26c10552ba20fb0dd76926cdcb66 Mon Sep 17 00:00:00 2001 From: Ben Steffensmeier Date: Tue, 20 May 2014 17:42:08 -0500 Subject: [PATCH] Omaha #3093 Add option to hprof tool to zero primitive arrays for compression. Former-commit-id: 15c7e3bb2d2a4ee74ec8d7acb5ad7e9b88bf57ee [formerly ede28797fd17d6433258c57ec56cb679a88bf76f] [formerly 555494618672990e377eae9f2e19d906010ae5ad] [formerly df6260997391e77f7ccc1848a2508f8e823cba33 [formerly 555494618672990e377eae9f2e19d906010ae5ad [formerly 6a80964572cf331b433ae8dc222ac209dd35bf63]]] Former-commit-id: df6260997391e77f7ccc1848a2508f8e823cba33 Former-commit-id: 958eb5b1366c7a3444b58acbc50cd6f3390ced96 [formerly b165082aad5d042bdf6c1949eaf68e2749d863ce] Former-commit-id: 0e8bd044cf4c23bfb1dea2eef05ae57d6c53b3b4 --- .../src/com/raytheon/hprof/BigByteBuffer.java | 54 +++++++++++++------ .../src/com/raytheon/hprof/HprofFile.java | 5 +- .../raytheon/hprof/data/HeapDumpRecord.java | 31 +++++++++-- .../hprof/data/heap/dump/ObjectArrayDump.java | 14 +++-- .../data/heap/dump/PrimitiveArrayDump.java | 17 ++++-- .../raytheon/uf/viz/hprof/CaveExporter.java | 19 +++++-- 6 files changed, 105 insertions(+), 35 deletions(-) diff --git a/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/BigByteBuffer.java b/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/BigByteBuffer.java index f7623cd34b..5bf43cd01a 100644 --- a/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/BigByteBuffer.java +++ b/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/BigByteBuffer.java @@ -19,8 +19,8 @@ **/ package com.raytheon.hprof; -import java.io.FileInputStream; import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; @@ -40,6 +40,7 @@ import java.nio.channels.FileChannel.MapMode; * Date Ticket# Engineer Description * ------------- -------- ----------- -------------------------- * Jan 08, 2014 2648 bsteffen Initial doc + * May 20, 2014 3093 bsteffen Allow read/write * * * @@ -53,30 +54,38 @@ public class BigByteBuffer { private final ByteBuffer[] buffers; private final long offset; - + private final long capacity; - + private long mark = -1; private long position = 0; private long limit; - public BigByteBuffer(String fileName) throws IOException { - FileInputStream fis = new FileInputStream(fileName); - FileChannel fileChannel = fis.getChannel(); - offset = 0; - limit = capacity = fileChannel.size(); - int nBuffers = (int) (capacity / chunkSize); - if (nBuffers * chunkSize < capacity) { - nBuffers += 1; + public BigByteBuffer(String fileName, boolean readOnly) throws IOException { + String mode = readOnly ? "r" : "rw"; + RandomAccessFile raf = null; + try { + raf = new RandomAccessFile(fileName, mode); + FileChannel fileChannel = raf.getChannel(); + offset = 0; + limit = capacity = fileChannel.size(); + int nBuffers = (int) (capacity / chunkSize); + if (nBuffers * chunkSize < capacity) { + nBuffers += 1; + } + buffers = new ByteBuffer[nBuffers]; + MapMode mapMode = readOnly ? MapMode.READ_ONLY : MapMode.READ_WRITE; + for (int i = 0; i < buffers.length; i += 1) { + buffers[i] = fileChannel.map(mapMode, i * chunkSize, + Math.min(capacity - i * chunkSize, chunkSize)); + } + } finally { + if (raf != null) { + raf.close(); + } } - buffers = new ByteBuffer[nBuffers]; - for (int i = 0; i < buffers.length; i += 1) { - buffers[i] = fileChannel.map(MapMode.READ_ONLY, i * chunkSize, - Math.min(capacity - i * chunkSize, chunkSize)); - } - fis.close(); } protected BigByteBuffer(ByteBuffer[] buffers, long offset, long capacity) { @@ -146,6 +155,17 @@ public class BigByteBuffer { return getBuffer(index).get(bufferIndex(index)); } + public BigByteBuffer put(byte b) { + put(nextGetIndex(), b); + return this; + } + + public BigByteBuffer put(long index, byte b) { + checkIndex(index); + getBuffer(index).put(bufferIndex(index), b); + return this; + } + public void get(byte[] dst) { long start = nextGetIndex(dst.length); ByteBuffer buffer = getBuffer(start); diff --git a/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/HprofFile.java b/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/HprofFile.java index b4f7a9688a..11e27b9a69 100644 --- a/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/HprofFile.java +++ b/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/HprofFile.java @@ -51,6 +51,7 @@ import com.raytheon.hprof.data.heap.dump.InstanceDump; * Date Ticket# Engineer Description * ------------- -------- ----------- -------------------------- * Jan 08, 2014 2648 bsteffen Initial doc + * May 20, 2014 3093 bsteffen Allow opening for write. * * * @@ -70,8 +71,8 @@ public class HprofFile { private Map usedClassnames = new HashMap(); - public HprofFile(String fileName) throws IOException { - BigByteBuffer buffer = new BigByteBuffer(fileName); + public HprofFile(String fileName, boolean readOnly) throws IOException { + BigByteBuffer buffer = new BigByteBuffer(fileName, readOnly); String format = null; diff --git a/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/data/HeapDumpRecord.java b/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/data/HeapDumpRecord.java index 853ea103c3..be65871267 100644 --- a/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/data/HeapDumpRecord.java +++ b/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/data/HeapDumpRecord.java @@ -55,6 +55,7 @@ import com.raytheon.hprof.data.heap.root.RootThreadObject; * Date Ticket# Engineer Description * ------------- -------- ----------- -------------------------- * Jan 08, 2014 2648 bsteffen Initial doc + * May 20, 2014 3093 bsteffen Add zeroPrimitiveArrays * * * @@ -137,12 +138,12 @@ public class HeapDumpRecord extends AbstractHprofRecord { } case ObjectArrayDump.TAG: { objectArrays.add(buffer.position()); - new ObjectArrayDump(buffer, idSize); + new ObjectArrayDump(buffer, idSize, false); break; } case PrimitiveArrayDump.TAG: { primitiveArrays.add(buffer.position()); - new PrimitiveArrayDump(buffer, idSize); + new PrimitiveArrayDump(buffer, idSize, false); break; } case HeapDumpSegmentRecord.TAG: { @@ -234,7 +235,7 @@ public class HeapDumpRecord extends AbstractHprofRecord { return null; } buffer.position(index); - return new ObjectArrayDump(buffer, idSize); + return new ObjectArrayDump(buffer, idSize, true); } public BasicType[] getPrimitiveArray(Id objectId) { @@ -243,7 +244,29 @@ public class HeapDumpRecord extends AbstractHprofRecord { return null; } buffer.position(index); - return new PrimitiveArrayDump(buffer, idSize).getArray(); + return new PrimitiveArrayDump(buffer, idSize, true).getArray(); + } + + /** + * Zero out all data for non char primitive arrays in the backing file. + */ + public void zeroPrimitiveArrays() { + for (long index : primitiveArraysById) { + buffer.position(index); + PrimitiveArrayDump dump = new PrimitiveArrayDump(buffer, idSize, + true); + BasicType[] array = dump.getArray(); + if (array.length > 0) { + BasicType sample = array[0]; + if (!sample.isChar()) { + int size = sample.getSize() * array.length; + buffer.position(buffer.position() - size); + for (int i = 0; i < size; i += 1) { + buffer.put((byte) 0); + } + } + } + } } private long binarySearch(long[] array, Id objectId) { diff --git a/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/data/heap/dump/ObjectArrayDump.java b/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/data/heap/dump/ObjectArrayDump.java index 9b8cdc34fd..733a03c49e 100644 --- a/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/data/heap/dump/ObjectArrayDump.java +++ b/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/data/heap/dump/ObjectArrayDump.java @@ -34,6 +34,7 @@ import com.raytheon.hprof.data.HeapDumpRecord; * Date Ticket# Engineer Description * ------------- -------- ----------- -------------------------- * Jan 08, 2014 2648 bsteffen Initial doc + * May 20, 2014 3093 bsteffen Allow creation without reading full array. * * * @@ -48,13 +49,18 @@ public class ObjectArrayDump extends AbstractDump { private final Id[] array; - public ObjectArrayDump(BigByteBuffer buffer, int idSize) { + public ObjectArrayDump(BigByteBuffer buffer, int idSize, boolean readArray) { super(buffer, idSize); int numElements = buffer.getInt(); arrayClassId = new Id(buffer, idSize); - array = new Id[numElements]; - for (int i = 0; i < numElements; i++) { - array[i] = new Id(buffer, idSize); + if (readArray) { + array = new Id[numElements]; + for (int i = 0; i < numElements; i++) { + array[i] = new Id(buffer, idSize); + } + } else { + array = null; + buffer.position(buffer.position() + idSize * numElements); } } diff --git a/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/data/heap/dump/PrimitiveArrayDump.java b/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/data/heap/dump/PrimitiveArrayDump.java index 1406cbf7f4..ea01ca5d43 100644 --- a/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/data/heap/dump/PrimitiveArrayDump.java +++ b/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/hprof/data/heap/dump/PrimitiveArrayDump.java @@ -34,6 +34,7 @@ import com.raytheon.hprof.data.heap.BasicType; * Date Ticket# Engineer Description * ------------- -------- ----------- -------------------------- * Jan 08, 2014 2648 bsteffen Initial doc + * May 20, 2014 3093 bsteffen Allow creation without reading full array. * * * @@ -46,13 +47,21 @@ public class PrimitiveArrayDump extends AbstractDump { private final BasicType[] array; - public PrimitiveArrayDump(BigByteBuffer buffer, int idSize) { + public PrimitiveArrayDump(BigByteBuffer buffer, int idSize, + boolean readArray) { super(buffer, idSize); int numElements = buffer.getInt(); byte type = buffer.get(); - array = new BasicType[numElements]; - for (int i = 0; i < numElements; i++) { - array[i] = new BasicType(buffer, type, idSize); + if (readArray) { + array = new BasicType[numElements]; + for (int i = 0; i < numElements; i++) { + array[i] = new BasicType(buffer, type, idSize); + } + } else { + array = null; + BasicType tmp = new BasicType(buffer, type, idSize); + int skip = (numElements - 1) * tmp.getSize(); + buffer.position(buffer.position() + skip); } } diff --git a/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/uf/viz/hprof/CaveExporter.java b/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/uf/viz/hprof/CaveExporter.java index 3ec4d12453..b6913781c3 100644 --- a/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/uf/viz/hprof/CaveExporter.java +++ b/javaUtilities/com.raytheon.uf.viz.hprof/src/com/raytheon/uf/viz/hprof/CaveExporter.java @@ -37,6 +37,7 @@ import com.raytheon.hprof.HprofFile; * ------------- -------- ----------- -------------------------- * Jan 08, 2014 2648 bsteffen Initial doc * May 05, 2014 3093 bsteffen Add some new exporters + * May 20, 2014 3093 bsteffen Add option to zero primitive arrays. * * * @@ -51,17 +52,27 @@ public class CaveExporter { .println("Provide some args, an hprof file is required, an output directory is optional."); System.exit(1); } - HprofFile hprof; + int argIdx = 0; + HprofFile hprof = null; try { - hprof = new HprofFile(args[0]); + if (args[argIdx].equals("-zero")) { + System.out.println("Writing 0s..."); + hprof = new HprofFile(args[argIdx + 1], false); + hprof.getHeapDump().zeroPrimitiveArrays(); + argIdx += 2; + } else { + hprof = new HprofFile(args[argIdx], true); + argIdx += 1; + } + } catch (IOException e) { throw new IllegalArgumentException("Error opening hprof file: " - + args[0], e); + + args[argIdx], e); } File outputDir = null; if (args.length > 1) { - outputDir = new File(args[1]); + outputDir = new File(args[argIdx++]); } else { outputDir = new File("."); }