xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Omaha #3499 added sparse array memory optimization to grid

Browse source 
Change-Id: I1db409a565e371fe5aa90f3dc7f921ea1004baa9

Former-commit-id: fcb130e3f8 [formerly 8dc30a64de [formerly 3b6b13dcf270aac9d19dee095dfd6bd78f13137d]]
Former-commit-id: 8dc30a64de
Former-commit-id: 9b4a50cef9
This commit is contained in:
Brian Clements 2014-08-06 16:22:39 -05:00
parent 41f9aa07f0
commit c6fa6d0103
1 changed files with 126 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

138
cave/com.raytheon.viz.grid/src/com/raytheon/viz/grid/rsc/general/GridMemoryManager.java
View file

@ -36,6 +36,9 @@ import org.geotools.coverage.grid.GridGeometry2D;
import com.raytheon.uf.common.numeric.DataUtilities;
import com.raytheon.uf.common.numeric.buffer.FloatBufferWrapper;
import com.raytheon.uf.common.numeric.source.DataSource;
import com.raytheon.uf.common.numeric.sparse.SparseArray;
import com.raytheon.uf.common.numeric.sparse.SparseFloatArray;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
@ -51,6 +54,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* Date Ticket# Engineer Description
* ------------- -------- ----------- --------------------------
* Feb 25, 2014 2791 bsteffen Initial creation
* Aug 06, 2014 3499 bclement added sparse array optimization
*
* </pre>
*
@ -77,6 +81,14 @@ public class GridMemoryManager {
* the available direct memory.
*/
private static final double DIRECT_MEMORY_PERCENTAGE = 0.75f;
private static final int SPARSE_CHECK_THRESHOLD = Integer.getInteger(
"grid.sparse.check.threshold", 100 * 1024 * 1024); // 100 MB
private static final int SPARSE_ARRAY_BLOCK_SIZE = Integer.getInteger(
"grid.sparse.block.size", SparseArray.DEFAULT_BLOCK_SIZE);
private static final float SPARSE_FILL_VALUE = Float.NaN;
private static GridMemoryManager instance;
@ -128,8 +140,8 @@ public class GridMemoryManager {
}
/**
* If there is free direct memory then copy data into direct memory.
* Otherwise just return data as is.
* Attempt to make memory optimizations if applicable. Otherwise just return
* data as is.
*
* @param data
* @return
@ -144,19 +156,31 @@ public class GridMemoryManager {
sizeInBytes *= 2;
}
long r = remainingDirectMemory.get();
boolean makeDirect = false;
while (r > sizeInBytes && makeDirect == false) {
if (remainingDirectMemory.compareAndSet(r, r - sizeInBytes)) {
makeDirect = true;
}
r = remainingDirectMemory.get();
/*
* TODO checking for memory optimizations could be moved to an external
* class so things like GFE could take advantage of it
*/
if (sizeInBytes > SPARSE_CHECK_THRESHOLD && isSparse(data)) {
data = convertToSparse(data);
} else if (hasAvailableDirectMemory(sizeInBytes)) {
data = convertToDirect(data, sizeInBytes, numGridPoints);
}
if (!makeDirect) {
return data;
}
return data;
}
/**
* Loads grid data into direct memory buffers
*
* @param data
* @param sizeInBytes
* @param numGridPoints
* @return a new grid data object with direct memory data sources
*/
private GeneralGridData convertToDirect(GeneralGridData data,
int sizeInBytes, int numGridPoints) {
GridGeometry2D gridGeometry = data.getGridGeometry();
GridEnvelope2D gridRange = gridGeometry.getGridRange2D();
try {
ByteBuffer directBuffer = ByteBuffer.allocateDirect(sizeInBytes);
livingRefs.add(new DirectBufferTrackingReference(directBuffer,
@ -193,6 +217,96 @@ public class GridMemoryManager {
return data;
}
/**
* @param sizeInBytes
* @return true if there is enough direct memory for the provided grid size
*/
private boolean hasAvailableDirectMemory(int sizeInBytes) {
long r = remainingDirectMemory.get();
boolean makeDirect = false;
while (r > sizeInBytes && makeDirect == false) {
if (remainingDirectMemory.compareAndSet(r, r - sizeInBytes)) {
makeDirect = true;
}
r = remainingDirectMemory.get();
}
return makeDirect;
}
/**
* @param data
* @param gridRange
* @return true if the data is mostly comprised of NaNs
*/
private boolean isSparse(GeneralGridData data) {
GridGeometry2D gridGeometry = data.getGridGeometry();
GridEnvelope2D gridRange = gridGeometry.getGridRange2D();
DataSource source;
if (data.isVector()) {
source = data.getUComponent();
} else {
source = data.getScalarData();
}
int nx = gridRange.width;
int ny = gridRange.height;
long nandCount = 0;
long largestNandRun = Long.MIN_VALUE;
long currentNandRun = 0;
boolean inRun = false;
for (int y = 0; y < ny; y += 1) {
for (int x = 0; x < nx; x += 1) {
if (Double.isNaN(source.getDataValue(x, y))) {
if (!inRun){
inRun = true;
}
nandCount += 1;
currentNandRun += 1;
} else {
if (inRun){
inRun = false;
largestNandRun = Math.max(largestNandRun, currentNandRun);
currentNandRun = 0;
}
}
}
}
return largestNandRun > SPARSE_ARRAY_BLOCK_SIZE
&& nandCount > ((nx * ny) / 4);
}
/**
* Loads data into a sparse arrays.
*
* @param data
* @return a new grid data object with sparse array data sources
*/
private GeneralGridData convertToSparse(GeneralGridData data) {
GridGeometry2D gridGeometry = data.getGridGeometry();
GridEnvelope2D gridRange = gridGeometry.getGridRange2D();
int nx = gridRange.width;
int ny = gridRange.height;
if (data.isVector()) {
DataSource uSource = data.getUComponent();
DataSource vSource = data.getVComponent();
SparseFloatArray uDest = new SparseFloatArray(nx, ny,
SPARSE_FILL_VALUE, SPARSE_ARRAY_BLOCK_SIZE);
SparseFloatArray vDest = new SparseFloatArray(nx, ny,
SPARSE_FILL_VALUE, SPARSE_ARRAY_BLOCK_SIZE);
DataUtilities.copy(uSource, uDest, nx, ny);
DataUtilities.copy(vSource, vDest, nx, ny);
data = GeneralGridData.createVectorDataUV(gridGeometry, uDest,
vDest, data.getDataUnit());
} else {
DataSource source = data.getScalarData();
SparseFloatArray dest = new SparseFloatArray(nx, ny,
SPARSE_FILL_VALUE, SPARSE_ARRAY_BLOCK_SIZE);
DataUtilities.copy(source, dest, nx, ny);
data = GeneralGridData.createScalarData(gridGeometry, dest,
data.getDataUnit());
}
return data;
}
private void free() {
DirectBufferTrackingReference ref = (DirectBufferTrackingReference) refQueue
.poll();