xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'omaha_13.5.1' (13.5.1-4) into development

Browse source 
Conflicts:
	edexOsgi/build.edex/build.xml
	edexOsgi/com.raytheon.edex.feature.uframe/feature.xml

Former-commit-id: 75c9b05bdd [formerly 2368697938] [formerly edeb3e255b] [formerly edeb3e255b [formerly 162261c395]] [formerly 76bc31aa77 [formerly edeb3e255b [formerly 162261c395] [formerly 76bc31aa77 [formerly 3ec3b75f720bc7622430f69a81ae566936be4363]]]]
Former-commit-id: 76bc31aa77
Former-commit-id: 3bf5b910e69ba4c0d0699130da6b05b74641b61b [formerly 86532f141a4385ff3262869d48ec5e99adacc4a2] [formerly f8213c16bd [formerly d3017ca60f]]
Former-commit-id: f8213c16bd
Former-commit-id: 5bcfa6048d
This commit is contained in:
Richard Peter 2013-07-01 00:04:43 -05:00
commit d246a809f5
32 changed files with 1084 additions and 467 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cave/build/p2-build.xml
View file

@ -264,6 +264,10 @@
<param name="feature"
value="com.raytheon.viz.volumebrowser.feature" />
</antcall>
<antcall target="p2.build.repo">
<param name="feature"
value="com.raytheon.uf.viz.archive.feature" />
</antcall>
<antcall target="p2.build.repo">
<param name="feature"
value="com.raytheon.uf.viz.d2d.gfe.feature" />

12
cave/com.raytheon.uf.viz.archive.feature/feature.xml
View file

@ -26,7 +26,11 @@
<import plugin="com.raytheon.uf.common.archive" version="1.0.0" match="greaterOrEqual"/>
<import plugin="com.raytheon.uf.common.time" version="1.12.1174" match="greaterOrEqual"/>
<import plugin="com.raytheon.uf.common.util" version="1.12.1174" match="greaterOrEqual"/>
<import plugin="org.apache.commons.io"/>
<import plugin="org.apache.commons.lang" version="2.3.0" match="greaterOrEqual"/>
<import plugin="com.raytheon.uf.viz.core" version="1.12.1174" match="greaterOrEqual"/>
<import plugin="com.raytheon.uf.common.units" version="1.0.0" match="greaterOrEqual"/>
<import plugin="com.raytheon.uf.common.status" version="1.12.1174" match="greaterOrEqual"/>
<import plugin="org.apache.commons.io" version="2.4.0" match="greaterOrEqual"/>
</requires>
<plugin
@ -43,4 +47,10 @@
version="0.0.0"
unpack="false"/>
<plugin
id="org.apache.commons.io"
download-size="0"
install-size="0"
version="0.0.0"/>
</feature>

85
cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/ui/AbstractArchiveDlg.java
View file

@ -24,6 +24,7 @@ import java.util.Calendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
@ -32,7 +33,6 @@ import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.swt.SWT;
import org.eclipse.swt.events.SelectionAdapter;
import org.eclipse.swt.events.SelectionEvent;
import org.eclipse.swt.graphics.Cursor;
import org.eclipse.swt.layout.GridData;
import org.eclipse.swt.layout.GridLayout;
import org.eclipse.swt.widgets.Combo;
@ -96,10 +96,19 @@ public abstract class AbstractArchiveDlg extends CaveSWTDialog implements
*/
protected boolean setSelect = false;
/**
* Must be set by sub-class prior to creating table.
*/
protected TableType tableType;
/**
* Job that computes sizes of table row entries off the UI thread.
*/
protected final SizeJob sizeJob = new SizeJob();
/** Keeps track of when it is safe to clear the busy cursor. */
protected final AtomicInteger busyCnt = new AtomicInteger(0);
/**
* @param parentShell
*/
@ -270,15 +279,27 @@ public abstract class AbstractArchiveDlg extends CaveSWTDialog implements
Job job = new Job("setup") {
@Override
protected IStatus run(IProgressMonitor monitor) {
initInfo();
VizApp.runAsync(new Runnable() {
ArchiveConfigManager.getInstance().reset();
if (!shell.isDisposed()) {
VizApp.runAsync(new Runnable() {
@Override
public void run() {
populateComboBoxes();
updateTableComp();
}
});
@Override
public void run() {
populateComboBoxes();
setCursorBusy(false);
}
});
}
initDisplayData();
if (!shell.isDisposed()) {
VizApp.runAsync(new Runnable() {
@Override
public void run() {
updateTableComp();
}
});
}
return Status.OK_STATUS;
}
};
@ -360,12 +381,11 @@ public abstract class AbstractArchiveDlg extends CaveSWTDialog implements
}
/**
* Get information from manager for populating combo boxes and set up to get
* selected entries sizes. Intended for use on a non-UI thread.
* Set up all display data and queue getting sizes for any that are
* selected.
*/
private void initInfo() {
private void initDisplayData() {
ArchiveConfigManager manager = ArchiveConfigManager.getInstance();
manager.reset();
Calendar startCal = getStart();
Calendar endCal = getEnd();
String[] archiveNames = manager.getArchiveDataNamesList();
@ -401,18 +421,32 @@ public abstract class AbstractArchiveDlg extends CaveSWTDialog implements
Calendar endCal = getEnd();
setCursorBusy(true);
sizeJob.clearQueue();
ArchiveInfo archiveInfo = archiveInfoMap.get(archiveName);
CategoryInfo categoryInfo = archiveInfo.get(categoryName);
try {
sizeJob.clearQueue();
for (DisplayData displayData : categoryInfo.getDisplayDataList()) {
sizeJob.queue(new SizeJobRequest(displayData, startCal, endCal));
ArchiveInfo archiveInfo = archiveInfoMap.get(archiveName);
// Not yet populated by background job.
if (archiveInfo == null) {
return;
}
// Not yet populated by background job.
CategoryInfo categoryInfo = archiveInfo.get(categoryName);
if (categoryInfo == null) {
return;
}
for (DisplayData displayData : categoryInfo.getDisplayDataList()) {
sizeJob.queue(new SizeJobRequest(displayData, startCal, endCal));
}
sizeJob.requeueSelected(startCal, endCal);
tableComp.populateTable(categoryInfo.getDisplayDataList());
} finally {
setCursorBusy(false);
}
sizeJob.requeueSelected(startCal, endCal);
tableComp.populateTable(categoryInfo.getDisplayDataList());
setCursorBusy(false);
}
/**
@ -421,11 +455,12 @@ public abstract class AbstractArchiveDlg extends CaveSWTDialog implements
* @param state
*/
protected void setCursorBusy(boolean state) {
Cursor cursor = null;
if (state) {
cursor = shell.getDisplay().getSystemCursor(SWT.CURSOR_WAIT);
busyCnt.addAndGet(1);
shell.setCursor(shell.getDisplay().getSystemCursor(SWT.CURSOR_WAIT));
} else if (busyCnt.addAndGet(-1) == 0) {
shell.setCursor(null);
}
shell.setCursor(cursor);
}
/*

1
cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/ui/ArchiveTableComp.java
View file

@ -380,6 +380,7 @@ public class ArchiveTableComp extends Composite {
}
updateSelectionLabels();
setModified();
}
/**

3
cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/ui/CaseCreationDlg.java
View file

@ -418,7 +418,7 @@ public class CaseCreationDlg extends AbstractArchiveDlg implements
browseBtn.setToolTipText("Select directory to place case.");
Label stateLbl = new Label(fileBrowserComp, SWT.NONE);
stateLbl.setText("Full - Avaliable:");
stateLbl.setText("Full - Available:");
locationStateLbl = new Label(fileBrowserComp, SWT.BORDER);
gd = new GridData(200, SWT.DEFAULT);
@ -612,6 +612,7 @@ public class CaseCreationDlg extends AbstractArchiveDlg implements
*/
private void handleBrowserSelection() {
DirectoryDialog dlg = new DirectoryDialog(shell, SWT.OPEN);
dlg.setText("Case Location");
String dirName = dlg.open();
if (dirName != null) {
locationLbl.setText(trimDirectoryName(dirName));

6
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/dialogs/FfmpBasinTableDlg.java
View file

@ -118,8 +118,9 @@ import com.raytheon.viz.ui.dialogs.ICloseCallback;
* Mar 29, 2013 1790 rferrel Bug fix for non-blocking dialogs.
* Apr 15, 2013 1904 mpduff Remove calls to reset FFMPConfig.
* Apr 25, 2013 1902 mpduff Fixed Thresholds dialog on multiple opens, needed an isDisposed check.
* Jun 04, 2013 #1984 lvenable removed unnecessary code.
* Jun 04, 2013 1984 lvenable removed unnecessary code.
* Jun 06, 2013 2075 njensen Removed loading labels
* Jun 20, 2013 14907 gzhang Thresholds menu ColorCell check fix.
* </pre>
*
* @author lvenable
@ -1086,8 +1087,7 @@ public class FfmpBasinTableDlg extends CaveSWTDialog implements
// Loop over enum from config singleton to create menu items
for (ThreshColNames colName : ThreshColNames.values()) {
if (ffmpConfig.isColorCell(colName)
&& (colName != ThreshColNames.GUID)) {// DR 14907
if (ffmpConfig.isColorCell(colName)) {// DR 14907
// only add a menu item if colorCell is true
MenuItem mi = new MenuItem(popupMenu, SWT.NONE);
mi.setText(colName.name());

5
cave/com.raytheon.viz.pointdata/src/com/raytheon/viz/pointdata/IPlotModelGeneratorCaller.java
View file

@ -29,7 +29,8 @@ import com.raytheon.uf.viz.core.drawables.IImage;
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Sep 16, 2009 mschenke Initial creation
* Sep 16, 2009 mschenke Initial creation
* Jun 25, 2013 1869 bsteffen Fix plot sampling.
*
* </pre>
*
@ -42,5 +43,5 @@ public interface IPlotModelGeneratorCaller {
public void clearImages();
public void messageGenerated(String dataURI, String message);
public void messageGenerated(PlotInfo[] key, String message);
}

85
cave/com.raytheon.viz.pointdata/src/com/raytheon/viz/pointdata/rsc/PlotResource2.java
View file

@ -28,7 +28,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang.Validate;
@ -94,6 +93,7 @@ import com.vividsolutions.jts.geom.Coordinate;
* May 14, 2013 1869 bsteffen Get plots working without dataURI
* Jun 06, 2013 2072 bsteffen Fix concurrency problems when init is
* called before time matching is done.
* Jun 25, 2013 1869 bsteffen Fix plot sampling.
*
* </pre>
*
@ -116,7 +116,7 @@ public class PlotResource2 extends
private double plotWidth;
private Map<DataTime, FrameInformation> frameMap;
private final Map<DataTime, FrameInformation> frameMap;
private DataTime displayedTime;
@ -125,8 +125,6 @@ public class PlotResource2 extends
private JobPool frameRetrievalPool = new JobPool("Retrieving plot frame",
8, true);
private TreeMap<String, String> rawMessageMap = new TreeMap<String, String>();
private class FrameRetriever implements Runnable {
private DataTime dataTime;
@ -144,13 +142,36 @@ public class PlotResource2 extends
}
/**
* A station represents all the data for a single location(station) for a
* single frame
*
*/
public static class Station {
/*
* Contains all PlotInfo objects for the same stationId with the same
* normalized time(real time will be different)
*/
public PlotInfo[] info;
/*
* The image to display for this plot
*/
public PointImage plotImage;
/*
* Sampling text for this plot
*/
public String rawMessage;
/*
* Information used be the progressive disclosure algorithm
*/
public Object progDiscInfo;
/*
* Location of the plot in descriptor grid space.
*/
public Coordinate pixelLocation;
}
@ -350,7 +371,7 @@ public class PlotResource2 extends
double[] thisLocationPixel = descriptor
.worldToPixel(new double[] {
plot.longitude, plot.latitude });
if (thisLocationPixel != null
if ((thisLocationPixel != null)
&& worldExtent.contains(
thisLocationPixel[0],
thisLocationPixel[1])) {
@ -385,10 +406,10 @@ public class PlotResource2 extends
existingStation.plotImage = null;
}
boolean dup = false;
for (int i = 0; i < existingStation.info.length; i++) {
String curUri = existingStation.info[i].dataURI;
for (PlotInfo element : existingStation.info) {
String curUri = element.dataURI;
String newUri = plot.dataURI;
if (curUri == null || curUri.equals(newUri)) {
if ((curUri == null) || curUri.equals(newUri)) {
dup = true;
break;
}
@ -494,9 +515,8 @@ public class PlotResource2 extends
for (Station station : stationMap.values()) {
if (station != null) {
Coordinate pixelLocation = station.pixelLocation;
if (distanceBetween(pixelLocation, mouseLocation) <= this
.getResourceData().getPixelSampleDistance()
/ screenToWorldRatio) {
if (distanceBetween(pixelLocation, mouseLocation) <= (this
.getResourceData().getPixelSampleDistance() / screenToWorldRatio)) {
availableStations.add(station);
}
@ -504,32 +524,25 @@ public class PlotResource2 extends
}
}
PlotInfo[] inspectPlot = null;
Station inspectPlot = null;
if (availableStations.size() == 1) {
inspectPlot = availableStations.get(0).info;
inspectPlot = availableStations.get(0);
} else if (availableStations.size() > 1) {
int index = findClosestPlot(latlon, availableStations);
if (index != -1) {
inspectPlot = availableStations.get(index).info;
inspectPlot = availableStations.get(index);
}
}
if (inspectPlot != null) {
String dataURI = inspectPlot[0].dataURI;
if (rawMessageMap.containsKey(dataURI)) {
if (rawMessageMap.get(dataURI) != null) {
message = rawMessageMap.get(dataURI);
}
} else {
message = inspectPlot.rawMessage;
if (message == null) {
message = "Generating...";
synchronized (rawMessageMap) {
rawMessageMap.put(dataURI, message);
}
List<PlotInfo[]> list = new ArrayList<PlotInfo[]>();
list.add(inspectPlot);
List<PlotInfo[]> list = new ArrayList<PlotInfo[]>(1);
list.add(inspectPlot.info);
Params params = Params.PLOT_AND_SAMPLE;
if (inspectPlot[0].pdv != null) {
if (inspectPlot.info[0].pdv != null) {
params = Params.SAMPLE_ONLY;
}
GetDataTask task = new GetDataTask(list, params);
@ -682,7 +695,7 @@ public class PlotResource2 extends
FrameInformation frameInfo = this.frameMap.remove(dataTime);
if (frameInfo != null) {
for (Station s : frameInfo.stationMap.values()) {
if (s != null && s.plotImage != null) {
if ((s != null) && (s.plotImage != null)) {
s.plotImage.getImage().dispose();
s.plotImage = null;
}
@ -709,7 +722,7 @@ public class PlotResource2 extends
Map<String, Station> stationMap = frame.stationMap;
List<PlotInfo[]> toQueue = new ArrayList<PlotInfo[]>(200);
for (Station station : disclosed) {
if (station.plotImage == null
if ((station.plotImage == null)
&& stationMap.containsKey(station.info[0].stationId)) {
toQueue.add(station.info);
}
@ -725,11 +738,19 @@ public class PlotResource2 extends
}
@Override
public void messageGenerated(String dataURI, String message) {
synchronized (rawMessageMap) {
rawMessageMap.put(dataURI, message);
public void messageGenerated(PlotInfo[] key, String message) {
// Key will be the same PlotInfo[] provided to the generator(which will
// be all the PlotInfo[] from a single station) and message will be the
// sample text.
DataTime normTime = getNormalizedTime(key[0].dataTime);
FrameInformation frameInfo = this.frameMap.get(normTime);
if (frameInfo != null) {
Station s = frameInfo.stationMap.get(key[0].stationId);
if (s != null) {
s.rawMessage = message;
issueRefresh();
}
}
issueRefresh();
}
}

3
cave/com.raytheon.viz.pointdata/src/com/raytheon/viz/pointdata/thread/PlotSampleGeneratorJob.java
View file

@ -43,6 +43,7 @@ import com.raytheon.viz.pointdata.PlotModelFactory2;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jul 13, 2011 njensen Initial creation
* Jun 25, 2013 1869 bsteffen Fix plot sampling.
*
* </pre>
*
@ -82,7 +83,7 @@ public class PlotSampleGeneratorJob extends Job {
String message = plotFactory.getStationMessage(infos[0].pdv,
infos[0].dataURI);
caller.messageGenerated(infos[0].dataURI, message);
caller.messageGenerated(infos, message);
} catch (Exception e) {
statusHandler.error("Error creating plot", e);
}

3
cave/com.raytheon.viz.ui.personalities.awips/META-INF/MANIFEST.MF
View file

@ -27,4 +27,5 @@ Bundle-ActivationPolicy: lazy
Bundle-ClassPath: .
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Export-Package: com.raytheon.viz.ui.personalities.awips
Import-Package: com.raytheon.uf.common.pypies
Import-Package: com.raytheon.uf.common.pypies,
com.raytheon.uf.viz.core.maps

17
cave/com.raytheon.viz.ui.personalities.awips/src/com/raytheon/viz/ui/personalities/awips/CAVE.java
View file

@ -19,6 +19,13 @@
**/
package com.raytheon.viz.ui.personalities.awips;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Status;
import org.eclipse.core.runtime.jobs.Job;
import com.raytheon.uf.viz.core.maps.MapStore;
/**
* TODO Add Description
*
@ -28,6 +35,7 @@ package com.raytheon.viz.ui.personalities.awips;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 9, 2010 mschenke Initial creation
* Jun 25, 2013 2139 jsanchez Loaded the map tree on CAVE start up.
*
* </pre>
*
@ -50,7 +58,14 @@ public class CAVE extends AbstractCAVEComponent {
*/
@Override
protected void startInternal(String componentName) throws Exception {
Job job = new Job("Loading Map Tree") {
@Override
protected IStatus run(IProgressMonitor monitor) {
MapStore.getMapTree();
return Status.OK_STATUS;
}
};
job.schedule();
}
}

4
edexOsgi/build.edex/build.xml
View file

@ -110,6 +110,10 @@
<param name="feature"
value="com.raytheon.uf.edex.dataprovideragent.feature" />
</antcall>
<antcall target="build">
<param name="feature"
value="com.raytheon.uf.edex.archive.feature" />
</antcall>
<!-- SPECIAL CASE -->
<if>

2
edexOsgi/build.edex/esb/conf/spring/cron.properties
View file

@ -14,6 +14,8 @@ gfe.cron=0+15+*+*+*+?
repack.cron=0+20+*+*+*+?
# runs database and hdf5 archive for archive server to pull data from
archive.cron=0+40+*+*+*+?
# purge archives
archive.purge.cron=0+0+*+*+*+?
###purge configuration
# Interval at which the purge job kicks off

4
edexOsgi/com.raytheon.edex.feature.uframe/feature.xml
View file

@ -132,4 +132,8 @@
id="com.raytheon.uf.edex.dataprovideragent.feature"
version="0.0.0"/>
<includes
id="com.raytheon.uf.edex.archive.feature"
version="0.0.0"/>
</feature>

2
edexOsgi/com.raytheon.uf.common.archive/build.properties
View file

@ -2,3 +2,5 @@ source.. = src/
output.. = bin/
bin.includes = META-INF/,\
.,\
/

277
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java
View file

@ -29,6 +29,7 @@ import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
@ -71,6 +72,9 @@ import com.raytheon.uf.common.util.FileUtil;
* ------------ ---------- ----------- --------------------------
* May 1, 2013 1966 rferrel Initial creation
* May 29, 2013 1965 bgonzale Added archive creation, purge, and save methods.
* Updated purgeExpiredFromArchive to check time of files in
* directory before purging them.
* Added null check for topLevelDirs in purgeExpiredFromArchive.
*
* </pre>
*
@ -203,23 +207,18 @@ public class ArchiveConfigManager {
public Collection<File> createArchive(File archiveDir,
Collection<DisplayData> displaysSelectedForArchive, Calendar start,
Calendar end) throws IOException, ArchiveException {
Collection<File> archivedFiles = null;
Collection<File> archivedFiles = new ArrayList<File>();
FileUtils.forceMkdir(archiveDir);
if (archiveDir.exists()) {
Collection<File> filesToArchive = new ArrayList<File>();
for (DisplayData display : displaysSelectedForArchive) {
List<File> files = getDisplayFiles(display, start, end);
filesToArchive.addAll(files);
String rootDirString = display.archiveConfig.getRootDir();
String archiveDirString = archiveDir.getAbsolutePath();
String rootDirString = display.archiveConfig.getRootDir();
archivedFiles = new ArrayList<File>(filesToArchive.size());
rootDirString = (rootDirString.endsWith(File.separator) ? rootDirString
.substring(0,
rootDirString.lastIndexOf(File.separatorChar))
: rootDirString);
for (File srcFile : filesToArchive) {
for (File srcFile : getDisplayFiles(display, start, end)) {
String fileAbsPath = srcFile.getAbsolutePath();
File newArchiveDir = getDirRelativeToArchiveDirFromRoot(
srcFile.isDirectory(), fileAbsPath, rootDirString,
@ -269,33 +268,87 @@ public class ArchiveConfigManager {
*/
public Collection<File> purgeExpiredFromArchive(ArchiveConfig archive) {
Collection<File> filesPurged = new ArrayList<File>();
String archiveRootDirPath = archive.getRootDir();
File archiveRootDir = new File(archiveRootDirPath);
String[] topLevelDirs = archiveRootDir.list();
List<String> topLevelDirsNotPurged = new ArrayList<String>();
if (topLevelDirs != null) {
topLevelDirsNotPurged.addAll(Arrays.asList(topLevelDirs));
}
for (CategoryConfig category : archive.getCategoryList()) {
Calendar purgeTime = calculateExpiration(archive, category);
CategoryFileDateHelper helper = new CategoryFileDateHelper(
category, archive.getRootDir());
IOFileFilter fileDateFilter = FileFilterUtils.and(
FileFilterUtils.fileFileFilter(),
new FileDateFilter(null, purgeTime, helper));
// Remove the directory associated with this category from the not
// purged list since it is being purged.
for (Iterator<String> iter = topLevelDirsNotPurged.iterator(); iter
.hasNext();) {
String dirName = iter.next();
if (helper.isCategoryDirectory(dirName)) {
iter.remove();
break;
}
}
for (DisplayData display : getDisplayData(archive.getName(),
category.getName(), true)) {
List<File> displayFiles = getDisplayFiles(display, null,
purgeTime);
for (File file : displayFiles) {
if (file.isFile()) {
filesPurged.add(file);
} else if (file.isDirectory()) {
filesPurged.addAll(FileUtils.listFiles(file,
FileFilterUtils.fileFileFilter(),
FileFilterUtils.trueFileFilter()));
}
FileUtils.deleteQuietly(file);
filesPurged.addAll(purgeFile(file, fileDateFilter,
archiveRootDirPath));
}
}
}
// check for other expired in top level directories not covered
// by the categories in the archives
Calendar defaultPurgeTime = calculateExpiration(archive, null);
for (String topDirName : topLevelDirsNotPurged) {
IOFileFilter fileDateFilter = FileFilterUtils.and(FileFilterUtils
.fileFileFilter(), new FileDateFilter(null,
defaultPurgeTime));
File topLevelDir = new File(archiveRootDir, topDirName);
filesPurged.addAll(purgeFile(topLevelDir, fileDateFilter,
archiveRootDirPath));
}
return filesPurged;
}
private Collection<File> purgeFile(File fileToPurge,
IOFileFilter filter, final String archiveRootDir) {
Collection<File> filesPurged = new ArrayList<File>();
if (fileToPurge.isFile() && filter.accept(fileToPurge)) {
filesPurged.add(fileToPurge);
FileUtils.deleteQuietly(fileToPurge);
} else if (fileToPurge.isDirectory()) {
Collection<File> expiredFilesInDir = FileUtils.listFiles(
fileToPurge, filter, FileFilterUtils.trueFileFilter());
for (File dirFile : expiredFilesInDir) {
filesPurged.addAll(purgeFile(dirFile, filter, archiveRootDir));
}
// if the directory is empty and not the archive root dir, then
// delete it
if (fileToPurge.list().length == 0
&& !fileToPurge.getAbsolutePath().equals(archiveRootDir)) {
FileUtils.deleteQuietly(fileToPurge);
}
}
return filesPurged;
}
private Calendar calculateExpiration(ArchiveConfig archive,
CategoryConfig category) {
Calendar newCal = TimeUtil.newGmtCalendar();
int retHours = category.getRetentionHours() == 0 ? archive
int retHours = category == null || category.getRetentionHours() == 0 ? archive
.getRetentionHours() : category.getRetentionHours();
if (retHours != 0) {
newCal.add(Calendar.HOUR, (-1) * retHours);
@ -462,40 +515,11 @@ public class ArchiveConfigManager {
List<File> fileList = new ArrayList<File>();
if (dirOnly) {
Pattern pattern = Pattern.compile(categoryConfig.getDirPattern());
for (String dirPattern : categoryConfig.getDirPatternList()) {
Pattern pattern = Pattern.compile(dirPattern);
for (File dir : dirs) {
String path = dir.getAbsolutePath().substring(beginIndex);
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
int year = Integer.parseInt(matcher.group(yearIndex));
// Adjust month value to Calendar's 0 - 11
int month = Integer.parseInt(matcher.group(monthIndex)) - 1;
int day = Integer.parseInt(matcher.group(dayIndex));
int hour = Integer.parseInt(matcher.group(hourIndex));
fileCal.set(year, month, day, hour, 0, 0);
long fileTime = fileCal.getTimeInMillis();
if ((startTime <= fileTime) && (fileTime < endTime)) {
fileList.add(dir);
}
}
}
} else {
Pattern pattern = Pattern.compile(categoryConfig.getDirPattern()
+ File.separator + categoryConfig.getFilePattern());
final Pattern filePattern = Pattern.compile("^" + filePatternStr
+ "$");
for (File dir : dirs) {
List<File> fList = FileUtil.listDirFiles(dir, new FileFilter() {
@Override
public boolean accept(File pathname) {
return filePattern.matcher(pathname.getName())
.matches();
}
}, false);
for (File file : fList) {
String path = file.getAbsolutePath().substring(beginIndex);
for (File dir : dirs) {
String path = dir.getAbsolutePath().substring(beginIndex);
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
int year = Integer.parseInt(matcher.group(yearIndex));
@ -506,7 +530,45 @@ public class ArchiveConfigManager {
fileCal.set(year, month, day, hour, 0, 0);
long fileTime = fileCal.getTimeInMillis();
if ((startTime <= fileTime) && (fileTime < endTime)) {
fileList.add(file);
fileList.add(dir);
}
}
}
}
} else {
for (String dirPattern : categoryConfig.getDirPatternList()) {
Pattern pattern = Pattern.compile(dirPattern + File.separator
+ categoryConfig.getFilePattern());
final Pattern filePattern = Pattern.compile("^"
+ filePatternStr + "$");
for (File dir : dirs) {
List<File> fList = FileUtil.listDirFiles(dir,
new FileFilter() {
@Override
public boolean accept(File pathname) {
return filePattern.matcher(
pathname.getName()).matches();
}
}, false);
for (File file : fList) {
String path = file.getAbsolutePath().substring(
beginIndex);
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
int year = Integer.parseInt(matcher
.group(yearIndex));
// Adjust month value to Calendar's 0 - 11
int month = Integer.parseInt(matcher
.group(monthIndex)) - 1;
int day = Integer.parseInt(matcher.group(dayIndex));
int hour = Integer.parseInt(matcher
.group(hourIndex));
fileCal.set(year, month, day, hour, 0, 0);
long fileTime = fileCal.getTimeInMillis();
if ((startTime <= fileTime) && (fileTime < endTime)) {
fileList.add(file);
}
}
}
}
@ -517,7 +579,7 @@ public class ArchiveConfigManager {
}
/**
* Get a list of directories matching the categories directory pattern that
* Get a list of directories matching the categories directory patterns that
* are sub-directories of the archive's root directory.
*
* @param archiveConfig
@ -526,35 +588,39 @@ public class ArchiveConfigManager {
*/
private List<File> getDirs(ArchiveConfig archiveConfig,
CategoryConfig categoryConfig) {
String dirPattern = categoryConfig.getDirPattern();
List<File> resultDirs = new ArrayList<File>();
File rootFile = new File(archiveConfig.getRootDir());
String[] subExpr = dirPattern.split(File.separator);
List<File> dirs = new ArrayList<File>();
dirs.add(rootFile);
List<File> tmpDirs = new ArrayList<File>();
List<File> swpDirs = null;
for (String regex : subExpr) {
Pattern subPattern = Pattern.compile("^" + regex + "$");
IOFileFilter filter = FileFilterUtils
.makeDirectoryOnly(new RegexFileFilter(subPattern));
for (File dir : dirs) {
File[] list = dir.listFiles();
if (list != null) {
List<File> dirList = Arrays.asList(list);
tmpDirs.addAll(Arrays.asList(FileFilterUtils.filter(filter,
dirList)));
}
}
swpDirs = dirs;
dirs = tmpDirs;
tmpDirs = swpDirs;
for (String dirPattern : categoryConfig.getDirPatternList()) {
String[] subExpr = dirPattern.split(File.separator);
dirs.clear();
dirs.add(rootFile);
tmpDirs.clear();
for (String regex : subExpr) {
Pattern subPattern = Pattern.compile("^" + regex + "$");
IOFileFilter filter = FileFilterUtils
.makeDirectoryOnly(new RegexFileFilter(subPattern));
for (File dir : dirs) {
File[] list = dir.listFiles();
if (list != null) {
List<File> dirList = Arrays.asList(list);
tmpDirs.addAll(Arrays.asList(FileFilterUtils.filter(
filter, dirList)));
}
}
swpDirs = dirs;
dirs = tmpDirs;
tmpDirs = swpDirs;
tmpDirs.clear();
}
resultDirs.addAll(dirs);
}
return dirs;
return resultDirs;
}
/**
@ -571,6 +637,18 @@ public class ArchiveConfigManager {
return getDisplayData(archiveName, categoryName, false);
}
/**
* Get the Display labels matching the pattern for the archive data's
* category. Assumes the archive data's root directory is the mount point to
* start the search.
*
* @param archiveName
* @param categoryName
* @param setSelect
* - when true set the displayData selection base on category's
* selection list
* @return displayDataList
*/
public List<DisplayData> getDisplayData(String archiveName,
String categoryName, boolean setSelect) {
Map<String, List<File>> displayMap = new HashMap<String, List<File>>();
@ -578,14 +656,20 @@ public class ArchiveConfigManager {
ArchiveConfig archiveConfig = archiveMap.get(archiveName);
CategoryConfig categoryConfig = findCategory(archiveConfig,
categoryName);
String dirPattern = categoryConfig.getDirPattern();
List<String> dirPatternList = categoryConfig.getDirPatternList();
// index for making directory paths' relative to the root path.
List<File> dirs = getDirs(archiveConfig, categoryConfig);
File rootFile = new File(archiveMap.get(archiveName).getRootDir());
int beginIndex = rootFile.getAbsolutePath().length() + 1;
Pattern pattern = Pattern.compile("^" + dirPattern + "$");
List<Pattern> patterns = new ArrayList<Pattern>(dirPatternList.size());
for (String dirPattern : dirPatternList) {
Pattern pattern = Pattern.compile("^" + dirPattern + "$");
patterns.add(pattern);
}
TreeSet<String> displays = new TreeSet<String>(
String.CASE_INSENSITIVE_ORDER);
@ -595,26 +679,31 @@ public class ArchiveConfigManager {
for (File dir : dirs) {
String path = dir.getAbsolutePath().substring(beginIndex);
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
sb.setLength(0);
String[] args = new String[matcher.groupCount() + 1];
args[0] = matcher.group();
for (int i = 1; i < args.length; ++i) {
args[i] = matcher.group(i);
for (Pattern pattern : patterns) {
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
sb.setLength(0);
String[] args = new String[matcher.groupCount() + 1];
args[0] = matcher.group();
for (int i = 1; i < args.length; ++i) {
args[i] = matcher.group(i);
}
String displayLabel = msgfmt.format(args, sb, pos0)
.toString();
List<File> displayDirs = displayMap.get(displayLabel);
if (displayDirs == null) {
displayDirs = new ArrayList<File>();
displayMap.put(displayLabel, displayDirs);
}
displayDirs.add(dir);
displays.add(displayLabel);
break;
}
String displayLabel = msgfmt.format(args, sb, pos0).toString();
List<File> displayDirs = displayMap.get(displayLabel);
if (displayDirs == null) {
displayDirs = new ArrayList<File>();
displayMap.put(displayLabel, displayDirs);
}
displayDirs.add(dir);
displays.add(displayLabel);
}
}
List<DisplayData> displayInfoList = new ArrayList<DisplayData>();
List<DisplayData> displayDataList = new ArrayList<DisplayData>(
displays.size());
for (String displayLabel : displays) {
DisplayData displayData = new DisplayData(archiveConfig,
@ -623,10 +712,10 @@ public class ArchiveConfigManager {
displayData.setSelected(categoryConfig
.getSelectedDisplayNames().contains(displayLabel));
}
displayInfoList.add(displayData);
displayDataList.add(displayData);
}
return displayInfoList;
return displayDataList;
}
/**

29
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java
View file

@ -19,7 +19,9 @@
**/
package com.raytheon.uf.common.archive.config;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.TreeSet;
import javax.xml.bind.annotation.XmlAccessType;
@ -96,7 +98,7 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
* </pre>
*/
@XmlElement(name = "dirPattern")
private String dirPattern;
private List<String> dirPatternList;
/**
* Use to display the information found by the dirPattern. Any groups in the
@ -187,21 +189,21 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
}
/**
* Obtain the directory pattern.
* Obtain the list of directory patterns.
*
* @return dirPattern
* @return dirPatternList
*/
public String getDirPattern() {
return dirPattern;
public List<String> getDirPatternList() {
return new ArrayList<String>(dirPatternList);
}
/**
* Set the directory pattern; must not be null.
* Set the directory pattern list; must not be null.
*
* @param dirPattern
* @param dirPatternList
*/
public void setDirPattern(String dirPattern) {
this.dirPattern = dirPattern;
public void setDirPatternList(List<String> dirPatternList) {
this.dirPatternList = dirPatternList;
}
/**
@ -285,7 +287,7 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
public int compareTo(CategoryConfig o) {
return getDisplay().compareToIgnoreCase(o.getDisplay());
}
/*
* (non-Javadoc)
*
@ -296,8 +298,11 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
StringBuilder sb = new StringBuilder();
sb.append("Category [ name: ").append(getName());
sb.append(", retentionHours: ").append(getRetentionHours());
sb.append(", dirPattern: ").append(getDirPattern());
sb.append(", filePattern: ").append(getFilePattern());
sb.append(", dirPatternList[ ");
for (String dirPattern : getDirPatternList()) {
sb.append(" \"").append(dirPattern).append("\",");
}
sb.append("], filePattern: ").append(getFilePattern());
sb.append(", displayLabel: ").append(getDisplay());
sb.append(", dateGroupIndices: ").append(getDateGroupIndices());
sb.append(", selectedDisplayNames: ");

196
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java Normal file
View file

@ -0,0 +1,196 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.archive.config;
import java.io.File;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FilenameUtils;
import com.raytheon.uf.common.time.util.TimeUtil;
/**
* File date helper for CategoryConfig objects.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 21, 2013 1965 bgonzale Initial creation
*
* </pre>
*
* @author bgonzale
* @version 1.0
*/
public class CategoryFileDateHelper implements IFileDateHelper {
/**
* Date information derived from each of a Category's dirPatterns.
*/
private static class CategoryDateInfo {
private final Pattern datePattern;
private final Pattern categoryTopLevelDirPattern;
private final int yearIndex;
private final int monthIndex;
private final int dayIndex;
private final int hourIndex;
/**
* Initialization constructor.
*
* @param datePattern
* @param categoryTopLevelDirPattern
* @param yearIndex
* @param monthIndex
* @param dayIndex
* @param hourIndex
*/
public CategoryDateInfo(Pattern datePattern,
Pattern categoryTopLevelDirPattern,
int yearIndex, int monthIndex, int dayIndex, int hourIndex) {
this.datePattern = datePattern;
this.categoryTopLevelDirPattern = categoryTopLevelDirPattern;
this.yearIndex = yearIndex;
this.monthIndex = monthIndex;
this.dayIndex = dayIndex;
this.hourIndex = hourIndex;
}
}
private final List<CategoryDateInfo> dateInfoList;
private final String rootDir;
private final boolean isDirOnly;
/**
* Initialization constructor.
*
* @param config
* @param rootDirPattern
* categoryTopLevelDirPattern
*/
public CategoryFileDateHelper(CategoryConfig config, String rootDir) {
List<String> categoryDirPatternList = config.getDirPatternList();
this.dateInfoList = new ArrayList<CategoryFileDateHelper.CategoryDateInfo>(
categoryDirPatternList.size());
String filePatternStr = config.getFilePattern();
this.rootDir = rootDir;
this.isDirOnly = (filePatternStr == null)
|| ".*".equals(filePatternStr);
for (String patternString : categoryDirPatternList) {
Pattern datePattern = null;
if (isDirOnly) {
datePattern = Pattern.compile(patternString);
} else {
datePattern = Pattern.compile(patternString + File.separator
+ config.getFilePattern());
}
int dirSeparatorIndex = patternString.indexOf(File.separatorChar);
patternString = dirSeparatorIndex > patternString.length()
|| dirSeparatorIndex < 0 ? patternString : patternString
.substring(0, dirSeparatorIndex);
Pattern categoryTopLevelDirPattern = Pattern.compile(patternString);
String[] indexValues = config.getDateGroupIndices().split(
"\\s*,\\s*");
int yearIndex = Integer.parseInt(indexValues[0]);
int monthIndex = Integer.parseInt(indexValues[1]);
int dayIndex = Integer.parseInt(indexValues[2]);
int hourIndex = Integer.parseInt(indexValues[3]);
dateInfoList.add(new CategoryDateInfo(datePattern,
categoryTopLevelDirPattern, yearIndex, monthIndex,
dayIndex, hourIndex));
}
}
/*
* (non-Javadoc)
*
* @see
* com.raytheon.uf.common.archive.config.IFileDateHelper#getFileDate(java
* .lang.String)
*/
@Override
public Calendar getFileDate(String filenamePath) {
String pathForPatternCheck = filenamePath.substring(rootDir.length());
pathForPatternCheck = isDirOnly ? FilenameUtils
.getFullPathNoEndSeparator(pathForPatternCheck)
: pathForPatternCheck;
Calendar result = null;
for (CategoryDateInfo dateInfo : dateInfoList) {
Matcher matcher = dateInfo.datePattern.matcher(pathForPatternCheck);
if (matcher.matches()) {
int year = Integer.parseInt(matcher.group(dateInfo.yearIndex));
// Adjust month value to Calendar's 0 - 11
int month = Integer
.parseInt(matcher.group(dateInfo.monthIndex)) - 1;
int day = Integer.parseInt(matcher.group(dateInfo.dayIndex));
int hour = Integer.parseInt(matcher.group(dateInfo.hourIndex));
result = TimeUtil.newGmtCalendar();
result.set(year, month, day, hour, 0, 0);
break;
}
}
if (result == null) {
// no matching pattern, use file last modified date
File file = new File(filenamePath);
long lastModifiedMillis = file.lastModified();
result = TimeUtil.newGmtCalendar();
result.setTimeInMillis(lastModifiedMillis);
}
return result;
}
/**
* Check if this directory is a category directory. i.e. if the category is
* satellite, is the directory satellite.
*
* @param dirName
* @return true if category directory; false otherwise.
*/
public boolean isCategoryDirectory(String dirName) {
for (CategoryDateInfo dateInfo : dateInfoList) {
if (dateInfo.categoryTopLevelDirPattern.matcher(dirName).matches()) {
return true;
}
}
return false;
}
}

125
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/FileDateFilter.java Normal file
View file

@ -0,0 +1,125 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.archive.config;
import java.io.File;
import java.util.Calendar;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import com.raytheon.uf.common.time.util.TimeUtil;
/**
* Filter files based on a file date parsed using the given file date helper.
* Accept returns true for files that fall between the Start and End times. If
* start is null, then all after start checks will return true. If end is null,
* then all before end checks will return true.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 18, 2013 1965 bgonzale Initial creation
*
* </pre>
*
* @author bgonzale
* @version 1.0
*/
public class FileDateFilter implements IOFileFilter {
private IFileDateHelper helper;
private final Calendar start;
private final Calendar end;
/**
* Initialization constructor. This filter uses file last modified time as
* the filter time.
*
* @param startDate
* @param endDate
*/
public FileDateFilter(Calendar start, Calendar end) {
this(start, end, DEFAULT_FILE_DATE_HELPER);
}
/**
* Initialization constructor.
*
* @param startDate
* @param endDate
* @param helper
*/
public FileDateFilter(Calendar start, Calendar end, IFileDateHelper helper) {
this.helper = helper == null ? DEFAULT_FILE_DATE_HELPER : helper;
this.start = start;
this.end = end;
}
/* (non-Javadoc)
* @see org.apache.commons.io.filefilter.IOFileFilter#accept(java.io.File)
*/
@Override
public boolean accept(File file) {
String filePath = file.getAbsolutePath();
String dirName = FilenameUtils.getFullPath(filePath);
String fileName = FilenameUtils.getName(filePath);
return accept(new File(dirName), fileName);
}
/* (non-Javadoc)
* @see org.apache.commons.io.filefilter.IOFileFilter#accept(java.io.File, java.lang.String)
*/
@Override
public boolean accept(File dir, String name) {
String dirPath = dir.getAbsolutePath();
boolean endsWithSeparator = dirPath.endsWith(File.separator);
dirPath = endsWithSeparator ? dirPath : dirPath + File.separator;
String fileFullPath = dirPath + name;
Calendar fileDate = helper.getFileDate(fileFullPath);
boolean isAfterEqualsStart = start == null || fileDate.after(start)
|| fileDate.equals(start);
boolean isBeforeEqualsEnd = end == null || fileDate.before(end)
|| fileDate.equals(end);
return isAfterEqualsStart && isBeforeEqualsEnd;
}
/**
* This File Date helper returns a file's last modified time.
*/
private static final IFileDateHelper DEFAULT_FILE_DATE_HELPER = new IFileDateHelper() {
@Override
public Calendar getFileDate(String filenamePath) {
// use file last modified date
File file = new File(filenamePath);
long lastModifiedMillis = file.lastModified();
Calendar result = TimeUtil.newGmtCalendar();
result.setTimeInMillis(lastModifiedMillis);
return result;
}
};
}

45
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/IFileDateHelper.java Normal file
View file

@ -0,0 +1,45 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.archive.config;
import java.util.Calendar;
/**
* Helper to get a file last modification date.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 21, 2013 bgonzale Initial creation
*
* </pre>
*
* @author bgonzale
* @version 1.0
*/
public interface IFileDateHelper {
public Calendar getFileDate(String filenamePath);
}

2
edexOsgi/com.raytheon.uf.edex.archive/res/spring/archivepurger-spring.xml
View file

@ -9,7 +9,7 @@
xmlns="http://camel.apache.org/schema/spring" errorHandlerRef="errorHandler">
<endpoint id="archivePurgeCron"
uri="clusteredquartz://archive/archivePurgeScheduled/?cron=${archivePurge.cron}" />
uri="clusteredquartz://archive/archivePurgeScheduled/?cron=${archive.purge.cron}" />
<!-- Run archivePurge on Scheduled timer -->
<route id="archivePurgeScheduled">

2
edexOsgi/com.raytheon.uf.edex.archive/resources/archivePurger.properties
View file

@ -1,2 +0,0 @@
# purge every half hour
archivePurge.cron=0+0/30,*,*,*,*,*+*+*+*+?

126
edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archive/PROCESSED_DATA.xml
View file

@ -18,6 +18,16 @@
See_the_AWIPS_II_Master_Rights_File_("Master_Rights_File.pdf")_for
further_licensing_information.
-->
<!--
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ============ ========== =========== ==========================
* Jun 20, 2013 1966 rferrel Initial creation
*
* @author rferrel
* @version 1.0
-->
<!--
The <archive> contains five types of tags:
<name> - Required tag. The id for the archive such as Raw or Processed.
@ -34,7 +44,8 @@
<dirPattern> - Required tag. A regex pattern for finding directories for this category.
The pattern is relative to the archive's <rootDir>. Wildcard patterns do not cross directory
delimiter /. Thus to match 3 levels of directories you would need .*/.*/.* .
See patterns and groups section.
See patterns and groups section. There may be more then one of these tags. The restriction is they
must all have the same number of groupings and be in the same order.
<filePattern> - Optional tag. A pattern to find files in the directories that match the <dirPattern>.
Default is everything in the directories that match <dirPattern>.
See patterns and groups section.
@ -73,6 +84,19 @@
<displayLabel>{1}</displayLabel>
<filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
Example with multiple <dirPattern>s
<category>
<name>Observation</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(acars|airep|airmet|taf)</dirPattern>
<dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</category>
The first <dirPattern> looks for files matching the <filePattern> in the directories acars, airep, airmet or taf.
The second <dirPattern> expects to find the files in subdirectories of bufrsigwx or sfcobs such as bufrsigwx/SWH.
Here the display will only show, redbook. The directory looked at will be <rootPath>/redbook/. The <dateGroupIndices> all
come from the <filePattern> since there is one group in the <dirPattern> the groups in the <filePattern> start at two. This
@ -85,68 +109,80 @@
<name>Processed</name>
<rootDir>/awips2/edex/data/archive/</rootDir>
<minRetentionHours>24</minRetentionHours>
<category>
<name>binlightning</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(binlightning)</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<category>
<name>DAT</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(cwat|fog|ffmp|preciprate|qpf|scan|vil)</dirPattern>
<filePattern>.*(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</category>
<category>
<name>bufr</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(bufrascat|bufrhdw|bufrmosLAMP|bufrncwf|bufrsigwx|bufrssmi|bufrua)</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<name>gfe</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(gfe)/(.*)/.*</dirPattern>
<dirPattern>(gfe)/(.*)</dirPattern>
<filePattern>.*_(\d{4})(\d{2})(\d{2})_(\d{2}).*</filePattern>
<displayLabel>{2}</displayLabel>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
</category>
<category>
<name>bufrsigwx</name>
<name>Model</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(bufrsigwx)/(.*)</dirPattern>
<displayLabel>{1} : {2}</displayLabel>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
<filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</category>
<category>
<name>group</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(acars|airmet|airep|cwa|cwat)</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</category>
<category>
<name>grib</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>grib/(.*)/(.*)</dirPattern>
<displayLabel>{1} : {2}</displayLabel>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
<dirPattern>(grid)/(.*)/(.*)</dirPattern>
<displayLabel>{2}</displayLabel>
<dateGroupIndices>4,5,6,7</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})-.*</filePattern>
</category>
<category>
<name>grid</name>
<name>Misc</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>grid/(.*)/(.*)</dirPattern>
<displayLabel>{1} : {2}</displayLabel>
<dirPattern>(binlightning|ccfp|cwa|idft|lsr|manual|svrwx|tcg|tcm|tcs|text|warning|wcp)</dirPattern>
<dirPattern>(redbook)/.*</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</category>
<category>
<name>Observation</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(acars|airep|airmet|bufrascat|bufrhdw|bufrmthdw|bufrncwf|bufrssmi|convsigmet|fssobs|intlsigmet|nonconvsigmet|obs|pirep|taf)</dirPattern>
<dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</category>
<category>
<name>Satellite</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>satellite/(.*)/(.*)</dirPattern>
<filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})-.*</filePattern>
<displayLabel>{1}</displayLabel>
</category>
<category>
<name>redbook</name>
<name>Sounding</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>redbook/(.*)</dirPattern>
<dirPattern>(acarssounding|bufrua|goessounding|poessounding|profiler|raobs)</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</category>
<category>
<name>obs</name>
<category>
<name>ModelSounding</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(fssobs|obs)</dirPattern>
<dirPattern>(modelsounding)/(.*)</dirPattern>
<dirPattern>(bufrmos)(.*)</dirPattern>
<displayLabel>{1} - {2}</displayLabel>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
<filePattern>.*(\d{4})-(\d{2})-(\d{2})[-_](\d{2}).*</filePattern>
</category>
<category>
<name>radar</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>radar/(.*)/(.*)</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
<filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</category>
</archive>

72
edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archive/RAW_DATA.xml
View file

@ -18,6 +18,16 @@
See_the_AWIPS_II_Master_Rights_File_("Master_Rights_File.pdf")_for
further_licensing_information.
-->
<!--
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ============ ========== =========== ==========================
* Jun 20, 2013 1966 rferrel Initial creation
*
* @author rferrel
* @version 1.0
-->
<!--
The <archive> contains five types of tags:
<name> - Required tag. The id for the archive such as Raw or Processed.
@ -34,7 +44,8 @@
<dirPattern> - Required tag. A regex pattern for finding directories for this category.
The pattern is relative to the archive's <rootDir>. Wildcard patterns do not cross directory
delimiter /. Thus to match 3 levels of directories you would need .*/.*/.* .
See patterns and groups section.
See patterns and groups section. There may be more then one of these tags. The restriction is they
must all have the same number of groupings and be in the same order.
<filePattern> - Optional tag. A pattern to find files in the directories that match the <dirPattern>.
Default is everything in the directories that match <dirPattern>.
See patterns and groups section.
@ -73,6 +84,19 @@
<displayLabel>{1}</displayLabel>
<filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
Example with multiple <dirPattern>s
<category>
<name>Observation</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(acars|airep|airmet|taf)</dirPattern>
<dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</category>
The first <dirPattern> looks for files matching the <filePattern> in the directories acars, airep, airmet or taf.
The second <dirPattern> expects to find the files in subdirectories of bufrsigwx or sfcobs such as bufrsigwx/SWH.
Here the display will only show, redbook. The directory looked at will be <rootPath>/redbook/. The <dateGroupIndices> all
come from the <filePattern> since there is one group in the <dirPattern> the groups in the <filePattern> start at two. This
@ -86,26 +110,41 @@
<rootDir>/data_store/</rootDir>
<minRetentionHours>24</minRetentionHours>
<category>
<name>Model grib</name>
<name>Model</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>grib/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<displayLabel>{5}</displayLabel>
<dateGroupIndices>1,2,3,4</dateGroupIndices>
</category>
<category>
<name>Model grib2</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<dirPattern>(grib|grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<displayLabel>{1} - {6}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</category>
<category>
<name>Model other</name>
<name>ModelSounding</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(acars|airmet|binlightning|bufrmos|bufrua|bufrsigwx|convsigmet|goessndg|manual|mdlsndg|poessndg|profiler|shef|text)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<dirPattern>(bufrmos|modelsounding)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</category>
<category>
<name>Misc</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(binlightning|climate|cwa|lsr|manual|misc_adm_messages|redbook|shef|summaries|svrwx|tcg|tcs|text|wwa|xml)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</category>
<category>
<name>Observation</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(airep|airmet|bufrascat|bufrhdw|bufrmthdw|bufrncwf|bufrsigwx|bufrssmi|convsigmet|fire_wx_spot_fcst_reports|forecast|fssobs|intlsigmet|MAROB|maritime|metar|misc_sfc_obs|nonconvsigmet|pirep|sfcobs|synoptic)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<dirPattern>acars/(.*)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</category>
<category>
<name>radar</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>radar/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<displayLabel>{5}</displayLabel>
<dateGroupIndices>1,2,3,4</dateGroupIndices>
</category>
<category>
<name>Satellite</name>
<extRetentionHours>168</extRetentionHours>
@ -113,4 +152,11 @@
<displayLabel>{5}</displayLabel>
<dateGroupIndices>1,2,3,4</dateGroupIndices>
</category>
</archive>
<category>
<name>Sounding</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(acarssounding|bufrua|goessounding|poessounding|profiler|raobs|upperair)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</category>
</archive>

38
edexOsgi/com.raytheon.uf.edex.maintenance/src/com/raytheon/uf/edex/maintenance/archive/DataArchiver.java
View file

@ -94,15 +94,6 @@ public class DataArchiver {
conf = defaultConf;
}
if (conf.getHoursToKeep() > 0) {
File pluginArchive = new File(baseArchive, pluginName);
if (pluginArchive.isDirectory()) {
long purgeThreshold = System.currentTimeMillis()
- (conf.getHoursToKeep() * 60 * 60 * 1000);
purgeDirectory(pluginArchive, purgeThreshold);
}
}
if (Boolean.TRUE.equals(conf.getArchivingEnabled())) {
for (IPluginArchiver pluginArchiver : pluginArchivers) {
pluginArchiver.archivePlugin(pluginName, archivePath, conf);
@ -134,33 +125,6 @@ public class DataArchiver {
return this;
}
private boolean purgeDirectory(File directory, long purgeThreshold) {
File[] listing = directory.listFiles();
int numDeleted = 0;
for (File file : listing) {
if (file.isDirectory()) {
if (purgeDirectory(file, purgeThreshold)) {
numDeleted++;
}
} else if (file.lastModified() < purgeThreshold) {
if (file.delete()) {
numDeleted++;
}
}
}
// we deleted all files/directories, or there were no files in this
// directory
if (numDeleted == listing.length
&& !directory.getAbsolutePath().equals(archivePath)) {
if (directory.delete()) {
return true;
}
}
return false;
}
private Map<String, DataArchiveConfig> getDataArchiveConfigs() {
Map<String, DataArchiveConfig> configs = new HashMap<String, DataArchiveConfig>();
IPathManager pathMgr = PathManagerFactory.getPathManager();
@ -232,8 +196,6 @@ public class DataArchiver {
if (!defaultConf.isHoursToKeepSet()) {
defaultConf.setHoursToKeep(6);
} else if (defaultConf.getHoursToKeep() < 0) {
}
// override unset fields with default

3
ncep/gov.noaa.nws.ncep.viz.rsc.plotdata/src/gov/noaa/nws/ncep/viz/rsc/plotdata/rsc/NcPlotResource2.java
View file

@ -119,6 +119,7 @@ import static java.lang.System.out;
* (this fixes the issue of slow performance when zooming all the way in, when Data Area is set)
* 10/18/2012 896 sgurung Refactored PlotResource2 to use new generator class: NcPlotDataThreadPool. Added FrameLoaderJob to populate all frames.
* Added code to plot stations within 25% of the area outside of the current display area.
* Jun 25, 2013 1869 bsteffen Fix plot sampling.
* </pre>
*
* @author brockwoo
@ -1232,7 +1233,7 @@ public class NcPlotResource2 extends AbstractNatlCntrsResource<PlotResourceData,
}
@Override
public void messageGenerated(String dataURI, String message) {
public void messageGenerated(PlotInfo[] key, String message) {
}
// generate a string used as the key for the StationMap

3
ncep/gov.noaa.nws.ncep.viz.rsc.plotdata/src/gov/noaa/nws/ncep/viz/rsc/plotdata/rsc/PlotResource2.java
View file

@ -117,6 +117,7 @@ import static java.lang.System.out;
* 08/22/2012 #809 sgurung For bgGenerator thread, add stations to queue only when zoomLevel > 0.10
* (this fixes the issue of slow performance when zooming all the way in, when Data Area is set)
* 11/04/2012 #944 ghull add query for Fcst Plot resources
* Jun 25, 2013 1869 bsteffen Fix plot sampling.
* </pre>
*
* @author brockwoo
@ -1084,7 +1085,7 @@ public class PlotResource2 extends AbstractNatlCntrsResource<PlotResourceData, N
}
@Override
public void messageGenerated(String dataURI, String message) {
public void messageGenerated(PlotInfo[] key, String message) {
}
// generate a string used as the key for the StationMap

2
rpms/awips2.core/Installer.ldm/component.spec
View file

@ -9,7 +9,7 @@
Name: awips2-ldm
Summary: AWIPS II LDM Distribution
Version: %{_ldm_version}
Release: 1
Release: 2
Group: AWIPSII
BuildRoot: /tmp
BuildArch: noarch

25
rpms/awips2.core/Installer.ldm/patch/etc/pqact.conf.template
View file

@ -33,6 +33,8 @@
#20121211 1411 rferrel More corrections and additions from Dale Morris.
#20130326 1828 rferrel Fix patterns for intlsigmets and text pattern not placing files in proper directory.
#20130530 2054 rferrel Minor fix to two grib patterns.
#20130624 1966 rferrel The acars pattern changed to place files in the proper subdirectories.
Name changes to reflect plugin names for modelsounding, goessounding, poessounding.
#***************************************************************
# AWIPS 1 PATTERN GRAPHIC ^[PQ].* /redbook/Raw
# PGNA00 KWNS 010001 !redbook 1_1/NMCGPHMCD/MCDSUM/PXSF001CN/20110201 0001
@ -386,7 +388,8 @@ IDS|DDPLUS ^(T[BCHPRTWXY][A-Z]{2}[0-9]{2}) ([A-Z]{4}) (..)(..)(..)
# Change submitted by Dale Morris
IDS|DDPLUS ^(A[AC-FH-RT-Z]..[0-9][0-9]) (.{4}) (..)(..)(..)
FILE -overwrite -log -close -edex /data_store/summaries/(\3:yyyy)(\3:mm)\3/\4/\1_\2_\3\4\5_(seq).%Y%m%d%H
IDS|DDPLUS ^(AG..[0-9][0-9]) (KWB.) (..)(..)(..)
#DR 16188 - Changed IDS|DDPLUS to ANY to handle products over both nwstg and nwstg2 channels
ANY ^(AG..[0-9][0-9]) (KWB.) (..)(..)(..)
FILE -overwrite -log -close -edex /data_store/summaries/(\3:yyyy)(\3:mm)\3/\4/\1_\2_\3\4\5_(seq).%Y%m%d%H
#shef
IDS|DDPLUS ^(A[BS]..[0-9][0-9]) (....) (..)(..)(..)
@ -609,15 +612,19 @@ HDS ^(JSM([TL]..|F1[0-7])) (....) (..)(..)(..)
# IUAX02 KARP 022359
ANY ^(IUAX0[12]) (....) (..)(..)(..)
PIPE -close /usr/local/ldm/decoders/decrypt_file
/data_store/acars/acars_raw/acars_decrypted_\1_\2_\3\4\5_(seq).%Y%m%d%H
FILE -overwrite -log -close /data_store/acars/acars_encrypted/(\3:yyyy)(\3:mm)\3/\4/\1_\2_\3\4\5_(seq).acars.%Y%m%d%H
ANY ^(IUAX0[12]) (....) (..)(..)(..)
PIPE -close /usr/local/ldm/decoders/decrypt_file
/data_store/acars/acars_decrypted/(\3:yyyy)(\3:mm)\3/\4/\1_\2_\3\4\5_(seq).acars.%Y%m%d%H
EXP ^/data_store/acars/acars_decrypted/(.*)
FILE -overwrite -log -close -edex /data_store/acars/acars_decrypted/\1
EXP ^/data_store/acars/acars_raw/(acars_decrypted.*)
FILE -overwrite -log -close -edex /data_store/acars/acars_decrypted/\1.acars.%Y%m%d%H
# Need to make sure that IUAK and IUAX are disallowed.
# IUAK are Alaskan profilers and IUAX has encrypted ACARS handled above!
ANY ^(IUA[^XK]0[12]) (....) (..)(..)(..)
FILE -overwrite -close -edex /data_store/acars/(\3:yyyy)(\3:mm)\3/\4/\1_\2_\3\4\5_(seq).bufr.%Y%m%d%H
FILE -overwrite -close -edex /data_store/acars/acars_raw_decrypted/(\3:yyyy)(\3:mm)\3/\4/\1_\2_\3\4\5_(seq).bufr.%Y%m%d%H
# AWIPS1: POINT ^JSAT98.*KKCI.* /aiv/ncwf/Raw
# JSAT98 KKCI 022210
@ -640,7 +647,7 @@ HDS ^(JU[BCFJMNOTVW]E(00|9[679])) KKCI (..)(..)(..)
# POINT ^JUSA41.KWBC* /ispan/bufr/modelSoundings/GFS
# JUSA42 KWNO 070200
HDS ^(JUS[ABX]4[1-9]) (KW(NO|BC)) (..)(..)(..)
FILE -overwrite -log -close -edex /data_store/mdlsndg/(\4:yyyy)(\4:mm)\4/\5/\1_\2_\4\5\6_(seq).bufr.%Y%m%d%H
FILE -overwrite -log -close -edex /data_store/modelsounding/(\4:yyyy)(\4:mm)\4/\5/\1_\2_\4\5\6_(seq).bufr.%Y%m%d%H
## GOES Bufr Patterns ##
# From goesBufrAcq_patterns.template
@ -650,7 +657,7 @@ HDS ^(JUS[ABX]4[1-9]) (KW(NO|BC)) (..)(..)(..)
#
# JUTX05 KNES 070326 RRN
HDS ^(JUTX0[1-9]) (....) (..)(..)(..)
FILE -overwrite -log -close -edex /data_store/goessndg/(\3:yyyy)(\3:mm)\3/\4/\1_\2_\3\4\5_(seq).bufr.%Y%m%d%H
FILE -overwrite -log -close -edex /data_store/goessounding/(\3:yyyy)(\3:mm)\3/\4/\1_\2_\3\4\5_(seq).bufr.%Y%m%d%H
## POES Bufr Patterns ##
# From poesBufrAcq_patterns.template
@ -659,7 +666,7 @@ HDS ^(JUTX0[1-9]) (....) (..)(..)(..)
# AWIPS1: POINT ^IUTX01.* /ispan/bufr/POESSoundings
# IUTX01 KNES 070307
HDS ^(IUTX0[1-9]) (....) (..)(..)(..)
FILE -overwrite -log -close -edex /data_store/poessndg/(\3:yyyy)(\3:mm)\3/\4/\1_\2_\3\4\5_(seq).bufr.%Y%m%d%H
FILE -overwrite -log -close -edex /data_store/poessounding/(\3:yyyy)(\3:mm)\3/\4/\1_\2_\3\4\5_(seq).bufr.%Y%m%d%H
## HDW Bufr Patterns ##
# From [east|west]HDWBufrAcq_patterns.template #

19
rpms/build/i386/build.sh
View file

@ -132,17 +132,15 @@ fi
if [ "${1}" = "-postgres" ]; then
buildRPM "awips2-postgres"
#buildRPM "awips2-database-server-configuration"
buildRPM "awips2-database-server-configuration"
buildRPM "awips2-database-standalone-configuration"
buildRPM "awips2-database"
buildRPM "awips2-maps-database"
buildRPM "awips2-pgadmin3"
buildRPM "awips2-tools"
buildRPM "awips2-notification"
buildRPM "awips2-edex-environment"
#buildRPM "awips2-database-standalone-configuration"
#buildRPM "awips2-database"
buildRPM "awips2-data.hdf5-gfe.climo"
buildRPM "awips2-data.hdf5-topo"
#buildRPM "awips2-maps-database"
buildRPM "awips2-python-dynamicserialize"
buildRPM "awips2-notification"
buildRPM "awips2-tools"
exit 0
fi
@ -350,7 +348,7 @@ fi
if [ "${1}" = "-viz" ]; then
buildRPM "awips2"
# buildRPM "awips2-common-base"
buildRPM "awips2-common-base"
buildRPM "awips2-rcm"
buildRPM "awips2-hydroapps-shared"
buildCAVE
@ -368,9 +366,6 @@ if [ "${1}" = "-edex" ]; then
buildRPM "awips2-cli"
buildRPM "awips2-gfesuite-client"
buildRPM "awips2-gfesuite-server"
buildRPM "awips2-python-dynamicserialize"
buildRPM "awips2-python-ufpy"
buildRPM "awips2-edex-environment"
buildEDEX
if [ $? -ne 0 ]; then
exit 1

264
tests/unit/com/raytheon/uf/common/archive/ArchiveConfigManagerTest.java
View file

@ -32,6 +32,7 @@ import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import org.apache.commons.io.FileUtils;
@ -60,7 +61,9 @@ import com.raytheon.uf.common.util.TestUtil;
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* May 7, 2013 1965 bgonzale Initial creation
* May 7, 2013 1965 bgonzale Initial creation.
* Added additional test data for file newer than purge
* time but in directory that is older than purge time.
*
* </pre>
*
@ -72,31 +75,29 @@ public class ArchiveConfigManagerTest {
private static final String RAW = "Raw";
private static final String SAT_CAT_NAME = "Satellite";
private static final String PROCESSED = "Processed";
private static final String satNameForArchive = "GOES-13";
private static final String SAT_CAT_NAME_RAW = "Satellite";
private static File TEST_DIR = TestUtil
.setupTestClassDir(ArchiveConfigManagerTest.class);
private final DateFormat yyyyMMFormat = new SimpleDateFormat("yyyyMM");
private final DateFormat yyyyFormat = new SimpleDateFormat("yyyy");
private final DateFormat MMFormat = new SimpleDateFormat("MM");
private final DateFormat ddFormat = new SimpleDateFormat("dd");
private final DateFormat hhFormat = new SimpleDateFormat("HH");
private final DateFormat kkFormat = new SimpleDateFormat("kk");
private final DateFormat mmFormat = new SimpleDateFormat("mm");
private ArchiveConfigManager manager;
private ArchiveConfig archive;
private Collection<File> archiveFiles = new ArrayList<File>();
private Collection<File> expiredFiles = new ArrayList<File>();
private Collection<File> purgeFiles = new ArrayList<File>();
private Collection<DisplayData> archiveSelectedDisplays = new HashSet<DisplayData>();
private Calendar referenceCalendar;
private Calendar archiveStart;
@ -134,55 +135,83 @@ public class ArchiveConfigManagerTest {
FileUtils.copyFileToDirectory(srcConfig, destDir);
}
manager = ArchiveConfigManager.getInstance();
archive = manager.getArchive(RAW);
ArchiveConfigManager manager = ArchiveConfigManager.getInstance();
// configure the test archive to use the test data dir
archive.setRootDir(TEST_DIR.getAbsolutePath() + "/data_store/");
ArchiveConfig archiveProcessed = manager.getArchive(PROCESSED);
archiveProcessed.setRootDir(TEST_DIR.getAbsolutePath() + "/archive/");
ArchiveConfig archiveRaw = manager.getArchive(RAW);
archiveRaw.setRootDir(TEST_DIR.getAbsolutePath() + "/data_store/");
// MessageFormat keys
// {0} yyyyMM {1} dd {2} hh {3} mm
// args:
// {0} dir-yyyy
// {1} dir-MM
// {2} dir-dd
// {3} dir-kk
// {4} file-yyyy
// {5} file-MM
// {6} file-dd
// {7} file-kk
// {8} file-mm
// **** grib1 ****
MessageFormat grib1Format = new MessageFormat(
"/grib/{0}{1}/18/NWS_160/GRID255/{2}{3}Z_F001_APCP-ZETA98_KSTR_{1}{2}{3}_125544891.grib.{0}{1}{2}");
CategoryConfig grib1Cat = getCategory(archive, "Model grib");
createTestFiles(grib1Format, getRetentionHours(archive, grib1Cat),
MessageFormat grib1Format_Raw = new MessageFormat(
"/grib/{0}{1}{2}/{3}/NWS_160/GRID255/{7}{8}Z_F001_APCP-ZETA98_KSTR_{6}{7}{8}_125544891.grib.{4}{5}{6}{7}");
createTestFiles(grib1Format_Raw, archiveRaw, "Model", false,
archiveStart, archiveEnd);
MessageFormat grib1Format_Processed = new MessageFormat(
"/grid/GFS160/BL/GFS160-{4}-{5}-{6}-{7}-FH-162.h5");
createTestFiles(grib1Format_Processed, archiveProcessed, "Model",
false, archiveStart, archiveEnd);
// **** sat ****
CategoryConfig satCat = getCategory(archive, SAT_CAT_NAME);
MessageFormat satFormat = new MessageFormat(
"/sat/{0}{1}/{2}/GOES-13/{2}{3}Z_SOUND-VIS_10km_EAST-CONUS-TIGE59_KNES_128453.satz.{0}{1}{2}");
createTestFiles(satFormat, getRetentionHours(archive, satCat), true,
MessageFormat satFormat_Raw = new MessageFormat(
"/sat/{0}{1}{2}/{3}/GOES-13/{7}{8}Z_SOUND-VIS_10km_EAST-CONUS-TIGE59_KNES_128453.satz.{4}{5}{6}{7}");
createTestFiles(satFormat_Raw, archiveRaw, SAT_CAT_NAME_RAW, true,
archiveStart, archiveEnd);
MessageFormat satFormat_Processed = new MessageFormat(
"/satellite/East CONUS/Sounder Visible imagery/satellite-{4}-{5}-{6}-{7}.h5");
createTestFiles(satFormat_Processed, archiveProcessed, "Satellite",
true, archiveStart, archiveEnd);
// **** acars ****
CategoryConfig otherCat = getCategory(archive, "Model other");
int otherCatRetentionHours = getRetentionHours(archive, otherCat);
MessageFormat acarsFormat = new MessageFormat(
"/acars/{0}{1}/{2}/IUAB01_CWAO_{1}{2}{3}_22714956.bufr.{0}{1}{2}");
createTestFiles(acarsFormat, otherCatRetentionHours, false,
MessageFormat acarsFormat_Raw = new MessageFormat(
"/acars/acars_encrypted/{0}{1}{2}/{3}/IUAB01_CWAO_{6}{7}{8}_22714956.bufr.{4}{5}{6}{7}");
createTestFiles(acarsFormat_Raw, archiveRaw, "Observation", false,
archiveStart, archiveEnd);
MessageFormat acarsFormat_Processed = new MessageFormat(
"/acars/acars-{4}-{5}-{6}-{7}.h5");
createTestFiles(acarsFormat_Processed, archiveProcessed, "Observation",
false, archiveStart, archiveEnd);
// **** binlightning ****
MessageFormat binlightningFormat = new MessageFormat(
"/binlightning/{0}{1}/{2}/SFUS41_KWBC_{1}{2}{3}_22725485.nldn.{0}{1}{2}");
createTestFiles(binlightningFormat, otherCatRetentionHours, false,
archiveStart, archiveEnd);
MessageFormat binlightningFormat_Raw = new MessageFormat(
"/binlightning/{0}{1}{2}/{3}/SFUS41_KWBC_{6}{7}{8}_22725485.nldn.{4}{5}{6}{7}");
createTestFiles(binlightningFormat_Raw, archiveRaw, "Misc",
false, archiveStart, archiveEnd);
MessageFormat binlightningFormat_Processed = new MessageFormat(
"/binlightning/binlightning-{4}-{5}-{6}-{7}.h5");
createTestFiles(binlightningFormat_Processed, archiveProcessed, "Misc",
false, archiveStart, archiveEnd);
// **** bufrsigwx ****
MessageFormat bufrsigwxFormat = new MessageFormat(
"/bufrsigwx/{0}{1}/{2}/JUWE96_KKCI_{1}{2}{3}_31368878.bufr.{0}{1}{2}");
createTestFiles(bufrsigwxFormat, otherCatRetentionHours, false,
MessageFormat bufrsigwxFormat_Raw = new MessageFormat(
"/bufrsigwx/{0}{1}{2}/{3}/JUWE96_KKCI_{6}{7}{8}_31368878.bufr.{4}{5}{6}{7}");
createTestFiles(bufrsigwxFormat_Raw, archiveRaw, "Observation", false,
archiveStart, archiveEnd);
MessageFormat bufrsigwxFormat_Processed = new MessageFormat(
"/bufrsigwx/SWH/sigwxCAT-{4}-{5}-{6}-{7}.h5");
createTestFiles(bufrsigwxFormat_Processed, archiveProcessed,
"Observation", false, archiveStart, archiveEnd);
// create test archive data dir
archiveDir = new File(TEST_DIR, TEST_ARCHIVE_DIR);
}
private int getRetentionHours(ArchiveConfig archive, CategoryConfig category) {
return category.getRetentionHours() == 0 ? archive.getRetentionHours()
return category == null || category.getRetentionHours() == 0 ? archive
.getRetentionHours()
: category.getRetentionHours();
}
@ -198,34 +227,83 @@ public class ArchiveConfigManagerTest {
return category;
}
private void createTestFiles(MessageFormat dataFileNameFormat,
int retentionHours, boolean isSelected, Calendar start, Calendar end)
throws IOException {
private void createTestFiles(MessageFormat fileNameFormat,
ArchiveConfig archive, String categoryName, boolean isSelected,
Calendar start, Calendar end) throws IOException {
CategoryConfig category = getCategory(archive, categoryName);
int retentionHours = getRetentionHours(archive, category);
String rootDir = archive.getRootDir();
// create data file newer than purge time, within archive time, and
// isSelected
File dataFile = createDataFile(end, dataFileNameFormat);
File dataFile = create_DataFile(end, fileNameFormat, rootDir);
if (isSelected) {
ArchiveConfigManager manager = ArchiveConfigManager.getInstance();
archiveFiles.add(dataFile);
archiveSelectedDisplays.addAll(manager.getDisplayData(
archive.getName(), categoryName, true));
}
System.out
.println("{newer than purge/within archive/isSelected}\n\tFor archive:"
+ archive.getName()
+ " category:"
+ categoryName
+ "\n\tcreated file: "
+ dataFile.getAbsolutePath()
.substring(rootDir.length()));
// create data file newer than purge time, within archive time, but not
// in selected
Calendar moreThanOneDayOld = (Calendar) referenceCalendar.clone();
moreThanOneDayOld.add(Calendar.DAY_OF_MONTH, -1);
createDataFile(moreThanOneDayOld, dataFileNameFormat);
dataFile = create_DataFile(moreThanOneDayOld, fileNameFormat, rootDir);
System.out
.println("{newer than purge/within archive/Not Selected}\nFor archive:"
+ archive.getName()
+ " category:"
+ categoryName
+ "\n\tcreated file: "
+ dataFile.getAbsolutePath()
.substring(rootDir.length()));
// create data file older than purge time
Calendar lessThanExpiredCalendar = (Calendar) referenceCalendar.clone();
lessThanExpiredCalendar.add(Calendar.HOUR, (-1 * retentionHours - 1));
dataFile = createDataFile(lessThanExpiredCalendar, dataFileNameFormat);
expiredFiles.add(dataFile);
dataFile = create_DataFile(lessThanExpiredCalendar, fileNameFormat,
rootDir);
purgeFiles.add(dataFile);
System.out.println("{older than purge}\nFor archive:"
+ archive.getName() + " category:" + categoryName
+ "\n\tcreated file: "
+ dataFile.getAbsolutePath().substring(rootDir.length()));
// // create data file newer than purge time, but in a directory that is
// // older than purge time, and outside of archive time frame
Calendar newerThanArchiveEnd = (Calendar) end.clone();
// newerThanArchiveEnd.add(Calendar.HOUR, 3);
// dataFile = create_DataFile(lessThanExpiredCalendar,
// newerThanArchiveEnd, fileNameFormat, rootDir);
// System.out
// .println("{newer than purge/in directory older than purge/outside of archive}\nFor archive:"
// + archive.getName()
// + " category:"
// + categoryName
// + "\n created file: " + dataFile.getAbsolutePath());
// create data file newer than purge time and outside of archive time
// frame
Calendar newerThanArchiveEnd = (Calendar) end.clone();
newerThanArchiveEnd = (Calendar) end.clone();
newerThanArchiveEnd.add(Calendar.HOUR, 3);
createDataFile(newerThanArchiveEnd, dataFileNameFormat);
dataFile = create_DataFile(newerThanArchiveEnd, fileNameFormat, rootDir);
System.out
.println("{newer than purge/outside of archive}\nFor archive:"
+ archive.getName()
+ " category:"
+ categoryName
+ "\n\tcreated file: "
+ dataFile.getAbsolutePath()
.substring(rootDir.length()));
}
private void setupTimes() {
@ -239,25 +317,40 @@ public class ArchiveConfigManagerTest {
archiveStart.add(Calendar.HOUR, -20);
archiveEnd.add(Calendar.HOUR, -3);
yyyyMMFormat.setCalendar(referenceCalendar);
yyyyFormat.setCalendar(referenceCalendar);
MMFormat.setCalendar(referenceCalendar);
ddFormat.setCalendar(referenceCalendar);
hhFormat.setCalendar(referenceCalendar);
kkFormat.setCalendar(referenceCalendar);
mmFormat.setCalendar(referenceCalendar);
}
private File createDataFile(Calendar referenceCalendar,
MessageFormat fileFormat) throws IOException {
Date referenceTime = referenceCalendar.getTime();
private File create_DataFile(Calendar referenceCalendar,
MessageFormat fileFormat, String rootDir) throws IOException {
return create_DataFile(referenceCalendar, referenceCalendar,
fileFormat, rootDir);
}
String yyyyMM = yyyyMMFormat.format(referenceTime);
String dd = ddFormat.format(referenceTime);
String hh = hhFormat.format(referenceTime);
String mm = mmFormat.format(referenceTime);
String[] formatArgs = new String[] { yyyyMM, dd, hh, mm };
private File create_DataFile(Calendar directoryReferenceCalendar,
Calendar fileReferenceCalendar, MessageFormat fileFormat,
String rootDir) throws IOException {
Date directoryReferenceTime = directoryReferenceCalendar.getTime();
Date fileReferenceTime = fileReferenceCalendar.getTime();
String dir_yyyy = yyyyFormat.format(directoryReferenceTime);
String dir_MM = MMFormat.format(directoryReferenceTime);
String dir_dd = ddFormat.format(directoryReferenceTime);
String dir_kk = kkFormat.format(directoryReferenceTime);
String file_yyyy = yyyyFormat.format(fileReferenceTime);
String file_MM = MMFormat.format(fileReferenceTime);
String file_dd = ddFormat.format(fileReferenceTime);
String file_kk = kkFormat.format(fileReferenceTime);
String file_mm = mmFormat.format(fileReferenceTime);
String[] formatArgs = new String[] { dir_yyyy, dir_MM, dir_dd, dir_kk,
file_yyyy, file_MM, file_dd, file_kk, file_mm };
String filename = fileFormat.format(formatArgs, new StringBuffer(),
new FieldPosition(0)).toString();
File resultFile = new File(archive.getRootDir(), filename);
File resultFile = new File(rootDir, filename);
String dirname = FilenameUtils
.getFullPath(resultFile.getAbsolutePath());
File dir = new File(dirname);
@ -267,13 +360,20 @@ public class ArchiveConfigManagerTest {
return resultFile;
}
private Collection<String> createFileNameListNoRootDir(File rootDir,
private Collection<String> createFileNameListRemoveTestDir(
Collection<File> files) {
List<String> result = new ArrayList<String>(files.size());
for (File f : files) {
String absPath = f.getAbsolutePath();
String fileRelativePath = absPath.replace(
rootDir.getAbsolutePath(), "");
String testDirPath = TEST_DIR.getAbsolutePath();
testDirPath = testDirPath.endsWith(File.separator) ? testDirPath
: testDirPath + File.separator;
// remove test directory path
String fileRelativePath = absPath.replace(testDirPath, "");
// remove one directory up
int index = fileRelativePath.indexOf(File.separator);
fileRelativePath = index >= 0 ? fileRelativePath.substring(index)
: fileRelativePath;
result.add(fileRelativePath);
}
Collections.sort(result);
@ -292,43 +392,39 @@ public class ArchiveConfigManagerTest {
@Test
public void testArchiveManagerCreateArchive() throws IOException,
ArchiveException {
CategoryConfig satCategory = getCategory(archive, SAT_CAT_NAME);
List<DisplayData> displays =
manager.getDisplayData(archive.getName(), satCategory.getName());
List<DisplayData> selectedDisplays = new ArrayList<DisplayData>();
for (DisplayData displayData : displays) {
if (displayData.getDisplayLabel().equals(satNameForArchive)) {
selectedDisplays.add(displayData);
}
}
Collection<File> archivedFiles = manager.createArchive(archiveDir,
selectedDisplays, archiveStart, archiveEnd);
ArchiveConfigManager manager = ArchiveConfigManager.getInstance();
Collection<File> filesInCreatedArchive = manager.createArchive(
archiveDir, archiveSelectedDisplays, archiveStart, archiveEnd);
assertEquals(
"The expected archive files and the archived files are not the same",
createFileNameListNoRootDir(new File(archive.getRootDir()),
archiveFiles),
createFileNameListNoRootDir(archiveDir, archivedFiles));
createFileNameListRemoveTestDir(archiveFiles),
createFileNameListRemoveTestDir(filesInCreatedArchive));
// check archive directory for files.
Collection<File> filesFoundInArchive = FileUtils.listFiles(archiveDir,
FileFilterUtils.trueFileFilter(),
FileFilterUtils.trueFileFilter());
assertEquals(
"The expected archive files in the files found in the archive are not the same",
archivedFiles, filesFoundInArchive);
"The archive files reported and the files found in the archive are not the same",
createFileNameListRemoveTestDir(filesInCreatedArchive),
createFileNameListRemoveTestDir(filesFoundInArchive));
}
@Test
public void testArchiveManagerPurge() throws IOException {
Collection<File> filesFoundInPurge = manager
.purgeExpiredFromArchive(archive);
// sort for comparison
List<File> purgeFilesList = new ArrayList<File>(purgeFiles);
Collections.sort(purgeFilesList);
List<File> foundFilesList = new ArrayList<File>(filesFoundInPurge);
Collections.sort(foundFilesList);
ArchiveConfigManager manager = ArchiveConfigManager.getInstance();
Collection<File> filesFoundInPurge = new ArrayList<File>();
for (ArchiveConfig a : manager.getArchives()) {
filesFoundInPurge.addAll(manager.purgeExpiredFromArchive(a));
}
assertEquals(
"The expected purge files and the files purged are not the same",
purgeFilesList, foundFilesList);
createFileNameListRemoveTestDir(purgeFiles),
createFileNameListRemoveTestDir(filesFoundInPurge));
}
}

87
tests/unit/com/raytheon/uf/common/archive/ArchiveManagerFactoryTest.java
View file

@ -1,87 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.archive;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
/**
* Test ArchiveManagerFactory.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* May 7, 2013 bgonzale Initial creation
*
* </pre>
*
* @author bgonzale
* @version 1.0
*/
public class ArchiveManagerFactoryTest {
/**
* @throws java.lang.Exception
*/
@BeforeClass
public static void setUpBeforeClass() throws Exception {
}
/**
* @throws java.lang.Exception
*/
@AfterClass
public static void tearDownAfterClass() throws Exception {
}
/**
* @throws java.lang.Exception
*/
@Before
public void setUp() throws Exception {
}
/**
* @throws java.lang.Exception
*/
@After
public void tearDown() throws Exception {
}
/**
* Test method for {@link com.raytheon.uf.common.archive.ArchiveManagerFactory#getManager()}.
*/
@Ignore
@Test
public void testGetManagerNotNull() {
// IArchiveManager manager = ArchiveManagerFactory.getManager();
// Assert.assertNotNull("ArchiveManagerFactory returned a null manager",
// manager);
}
}