xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2299 fixed purger running ingest out of memory.

Browse source 
Change-Id: If1877049574fd546e90479f3c56feb94ffbe088a

Former-commit-id: 79d8c93563 [formerly f75af60ec46f734bd5e815eeeab2bc82d3436229]
Former-commit-id: 1d71cf8beb
This commit is contained in:
Roger Ferrel 2013-08-28 10:33:56 -05:00 • committed by Richard Peter
parent 1467971c86
commit 7d9afba0db
6 changed files with 107 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java
View file

@ -87,7 +87,7 @@ import com.raytheon.uf.common.util.FileUtil;
* Added warn logging for failure to delete.
* Jul 24, 2013 2221 rferrel Changes for select configuration.
* Aug 06, 2013 2224 rferrel Changes to use DataSet.
*
* Aug 28, 2013 2299 rferrel purgeExpiredFromArchive now returns the number of files purged.
* </pre>
*
* @author rferrel
@ -288,18 +288,22 @@ public class ArchiveConfigManager {
* Archive.
*
* @param archive
* @return the list of expired Files purged
* @return purgeCount
*/
public Collection<File> purgeExpiredFromArchive(ArchiveConfig archive) {
Collection<File> filesPurged = new ArrayList<File>();
public int purgeExpiredFromArchive(ArchiveConfig archive) {
String archiveRootDirPath = archive.getRootDir();
File archiveRootDir = new File(archiveRootDirPath);
String[] topLevelDirs = archiveRootDir.list();
List<String> topLevelDirsNotPurged = new ArrayList<String>();
int purgeCount = 0;
if (topLevelDirs != null) {
topLevelDirsNotPurged.addAll(Arrays.asList(topLevelDirs));
topLevelDirs = null;
}
for (CategoryConfig category : archive.getCategoryList()) {
Calendar purgeTime = calculateExpiration(archive, category);
CategoryFileDateHelper helper = new CategoryFileDateHelper(
@ -323,68 +327,92 @@ public class ArchiveConfigManager {
List<File> displayFiles = getDisplayFiles(display, null,
purgeTime);
for (File file : displayFiles) {
filesPurged.addAll(purgeFile(file, fileDateFilter,
archiveRootDirPath));
purgeCount += purgeFile(file, fileDateFilter);
}
}
}
// check for other expired in top level directories not covered
// by the categories in the archives
// by the categories in the archive.
Calendar defaultPurgeTime = calculateExpiration(archive, null);
IOFileFilter fileDateFilter = FileFilterUtils.and(FileFilterUtils
.fileFileFilter(), new FileDateFilter(null, defaultPurgeTime));
for (String topDirName : topLevelDirsNotPurged) {
IOFileFilter fileDateFilter = FileFilterUtils.and(FileFilterUtils
.fileFileFilter(), new FileDateFilter(null,
defaultPurgeTime));
File topLevelDir = new File(archiveRootDir, topDirName);
filesPurged.addAll(purgeFile(topLevelDir, fileDateFilter,
archiveRootDirPath));
// Keep both top level hidden files and hidden directories.
if (!topLevelDir.isHidden()) {
purgeCount += purgeFile(topLevelDir, fileDateFilter);
}
}
return filesPurged;
return purgeCount;
}
private Collection<File> purgeFile(File fileToPurge, IOFileFilter filter,
final String archiveRootDir) {
Collection<File> filesPurged = new ArrayList<File>();
/**
* Recursive method for purging files. Never pass in a directory you do not
* want deleted when purging makes it an empty directory.
*
* @param fileToPurge
* @param filter
* @return purgeCount number of files and directories purged
*/
private int purgeFile(File fileToPurge, IOFileFilter filter) {
int purgeCount = 0;
if (fileToPurge.isFile() && filter.accept(fileToPurge)) {
if (fileToPurge.delete()) {
filesPurged.add(fileToPurge);
++purgeCount;
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.debug("Purged file: \""
+ fileToPurge.getAbsolutePath() + "\"");
}
} else {
statusHandler.warn("Failed to purge file: "
+ fileToPurge.getAbsolutePath());
}
} else if (fileToPurge.isDirectory()) {
Collection<File> expiredFilesInDir = FileUtils.listFiles(
fileToPurge, filter, FileFilterUtils.trueFileFilter());
} else if (fileToPurge.isDirectory() && !fileToPurge.isHidden()) {
// Purge only visible directories.
File[] expiredFilesInDir = fileToPurge.listFiles();
for (File dirFile : expiredFilesInDir) {
filesPurged.addAll(purgeFile(dirFile, filter, archiveRootDir));
purgeCount += purgeFile(dirFile, filter);
}
// if the directory is empty and not the archive root dir, then
// delete it
if (fileToPurge.list().length == 0
&& !fileToPurge.getAbsolutePath().equals(archiveRootDir)) {
// Attempt to delete empty directory.
if ((purgeCount >= expiredFilesInDir.length)
&& (fileToPurge.list().length == 0)) {
if (!fileToPurge.delete()) {
statusHandler.warn("Failed to purge directory: "
+ fileToPurge.getAbsolutePath());
} else {
++purgeCount;
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.debug("Purged directory: \""
+ fileToPurge.getAbsolutePath()
+ File.separator + "\"");
}
}
}
}
return filesPurged;
return purgeCount;
}
/**
* Get expiration time for the category.
*
* @param archive
* @param category
* @return expireCal
*/
private Calendar calculateExpiration(ArchiveConfig archive,
CategoryConfig category) {
Calendar newCal = TimeUtil.newGmtCalendar();
Calendar expireCal = TimeUtil.newGmtCalendar();
int retHours = category == null || category.getRetentionHours() == 0 ? archive
.getRetentionHours() : category.getRetentionHours();
if (retHours != 0) {
newCal.add(Calendar.HOUR, (-1) * retHours);
expireCal.add(Calendar.HOUR, (-1) * retHours);
}
return newCal;
return expireCal;
}
/**

7
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java
View file

@ -41,6 +41,7 @@ import com.raytheon.uf.common.time.util.TimeUtil;
* ------------ ---------- ----------- --------------------------
* Jun 21, 2013 1965 bgonzale Initial creation
* Aug 03, 2013 2224 rferrel Changes for new configuration files.
* Aug 28, 2013 2299 rferrel Changes in IFileDateHelper.
*
* </pre>
*
@ -137,10 +138,11 @@ public class CategoryFileDateHelper implements IFileDateHelper {
*
* @see
* com.raytheon.uf.common.archive.config.IFileDateHelper#getFileDate(java
* .lang.String)
* .io.File)
*/
@Override
public Calendar getFileDate(String filenamePath) {
public Calendar getFileDate(File file) {
String filenamePath = file.getAbsolutePath();
String pathForFilePatternCheck = filenamePath.substring(rootDir
.length());
String pathForDirPatternCheck = FilenameUtils
@ -165,7 +167,6 @@ public class CategoryFileDateHelper implements IFileDateHelper {
if (timestamp == null) {
// no matching pattern, use file last modified date
File file = new File(filenamePath);
timestamp = file.lastModified();
}

22
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/FileDateFilter.java
View file

@ -40,6 +40,7 @@ import com.raytheon.uf.common.time.util.TimeUtil;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 18, 2013 1965 bgonzale Initial creation
* Aug 28, 2013 2299 rferrel Reject hidden directories.
*
* </pre>
*
@ -79,7 +80,9 @@ public class FileDateFilter implements IOFileFilter {
this.end = end;
}
/* (non-Javadoc)
/*
* (non-Javadoc)
*
* @see org.apache.commons.io.filefilter.IOFileFilter#accept(java.io.File)
*/
@Override
@ -90,16 +93,16 @@ public class FileDateFilter implements IOFileFilter {
return accept(new File(dirName), fileName);
}
/* (non-Javadoc)
* @see org.apache.commons.io.filefilter.IOFileFilter#accept(java.io.File, java.lang.String)
/*
* (non-Javadoc)
*
* @see org.apache.commons.io.filefilter.IOFileFilter#accept(java.io.File,
* java.lang.String)
*/
@Override
public boolean accept(File dir, String name) {
String dirPath = dir.getAbsolutePath();
boolean endsWithSeparator = dirPath.endsWith(File.separator);
dirPath = endsWithSeparator ? dirPath : dirPath + File.separator;
String fileFullPath = dirPath + name;
Calendar fileDate = helper.getFileDate(fileFullPath);
File file = new File(dir, name);
Calendar fileDate = helper.getFileDate(file);
boolean isAfterEqualsStart = start == null || fileDate.after(start)
|| fileDate.equals(start);
boolean isBeforeEqualsEnd = end == null || fileDate.before(end)
@ -112,9 +115,8 @@ public class FileDateFilter implements IOFileFilter {
*/
private static final IFileDateHelper DEFAULT_FILE_DATE_HELPER = new IFileDateHelper() {
@Override
public Calendar getFileDate(String filenamePath) {
public Calendar getFileDate(File file) {
// use file last modified date
File file = new File(filenamePath);
long lastModifiedMillis = file.lastModified();
Calendar result = TimeUtil.newGmtCalendar();
result.setTimeInMillis(lastModifiedMillis);

10
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/IFileDateHelper.java
View file

@ -19,6 +19,7 @@
**/
package com.raytheon.uf.common.archive.config;
import java.io.File;
import java.util.Calendar;
/**
@ -31,6 +32,7 @@ import java.util.Calendar;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 21, 2013 bgonzale Initial creation
* Aug 28, 2013 2299 rferrel Change getFileDate argument.
*
* </pre>
*
@ -40,6 +42,12 @@ import java.util.Calendar;
public interface IFileDateHelper {
public Calendar getFileDate(String filenamePath);
/**
* Get data associated with the file.
*
* @param file
* @return calendar
*/
public Calendar getFileDate(File file);
}

10
edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/purge/ArchivePurger.java
View file

@ -19,7 +19,6 @@
**/
package com.raytheon.uf.edex.archive.purge;
import java.io.File;
import java.util.Collection;
import com.raytheon.uf.common.archive.config.ArchiveConfig;
@ -39,6 +38,8 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* ------------ ---------- ----------- --------------------------
* May 6, 2013 1965 bgonzale Initial creation
* Added info logging for purge counts.
* Aug 28, 2013 2299 rferrel manager.purgeExpiredFromArchive now returns
* number of files purged.
*
* </pre>
*
@ -57,14 +58,13 @@ public class ArchivePurger {
ArchiveConfigManager manager = ArchiveConfigManager.getInstance();
Collection<ArchiveConfig> archives = manager.getArchives();
for (ArchiveConfig archive : archives) {
Collection<File> deletedFiles = manager
.purgeExpiredFromArchive(archive);
int purgeCount = manager.purgeExpiredFromArchive(archive);
if (statusHandler.isPriorityEnabled(Priority.INFO)) {
StringBuilder sb = new StringBuilder(archive.getName());
sb.append("::Archive Purged ");
sb.append(deletedFiles.size());
sb.append(purgeCount);
sb.append(" file");
if (deletedFiles.size() != 1) {
if (purgeCount != 1) {
sb.append("s");
}
sb.append(".");

40
tests/unit/com/raytheon/uf/common/archive/ArchiveConfigManagerTest.java
View file

@ -64,6 +64,7 @@ import com.raytheon.uf.common.util.TestUtil;
* May 7, 2013 1965 bgonzale Initial creation.
* Added additional test data for file newer than purge
* time but in directory that is older than purge time.
* Aug 28, 2013 2299 rferrel purgeExpiredFromArchive now returns number of files purged.
*
* </pre>
*
@ -91,11 +92,13 @@ public class ArchiveConfigManagerTest {
private final DateFormat mmFormat = new SimpleDateFormat("mm");
private Collection<File> archiveFiles = new ArrayList<File>();
private final Collection<File> archiveFiles = new ArrayList<File>();
private Collection<File> purgeFiles = new ArrayList<File>();
private final Collection<File> purgeFiles = new ArrayList<File>();
private Collection<DisplayData> archiveSelectedDisplays = new HashSet<DisplayData>();
private final Collection<File> allFiles = new ArrayList<File>();
private final Collection<DisplayData> archiveSelectedDisplays = new HashSet<DisplayData>();
private Calendar referenceCalendar;
@ -115,25 +118,9 @@ public class ArchiveConfigManagerTest {
if (referenceCalendar == null) {
setupTimes();
}
File testLocalization = TestUtil
.setupTestClassDir(PathManagerFactoryTest.class);
PathManagerFactoryTest.initLocalization();
// after setting up test localization get the production config files
// and copy them to the the test localization directory.
// utility/common_static/base/archive
File prodConfigDir = new File(
"../edexOsgi/com.raytheon.uf.edex.archive/utility");
Collection<File> configs = FileUtils.listFiles(prodConfigDir,
FileFilterUtils.trueFileFilter(),
FileFilterUtils.trueFileFilter());
File destDir = new File(testLocalization,
"utility/common_static/base/archive");
for (File srcConfig : configs) {
FileUtils.copyFileToDirectory(srcConfig, destDir);
}
ArchiveConfigManager manager = ArchiveConfigManager.getInstance();
ArchiveConfig archiveProcessed = manager.getArchive(PROCESSED);
@ -355,6 +342,7 @@ public class ArchiveConfigManagerTest {
dir.mkdirs();
resultFile.createNewFile();
allFiles.add(resultFile);
return resultFile;
}
@ -414,9 +402,21 @@ public class ArchiveConfigManagerTest {
public void testArchiveManagerPurge() throws IOException {
ArchiveConfigManager manager = ArchiveConfigManager.getInstance();
Collection<File> filesFoundInPurge = new ArrayList<File>();
int purgeCount = 0;
for (ArchiveConfig a : manager.getArchives()) {
filesFoundInPurge.addAll(manager.purgeExpiredFromArchive(a));
purgeCount += manager.purgeExpiredFromArchive(a);
}
// assertEquals(
//
// "The expected number of purged files and number of purge files not the same",
// purgeCount, purgeFiles.size());
for (File file : allFiles) {
if (!file.exists()) {
filesFoundInPurge.add(file);
}
}
assertEquals(