diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java index f2eb0aa669..7c7c470ed5 100644 --- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java @@ -87,7 +87,7 @@ import com.raytheon.uf.common.util.FileUtil; * Added warn logging for failure to delete. * Jul 24, 2013 2221 rferrel Changes for select configuration. * Aug 06, 2013 2224 rferrel Changes to use DataSet. - * + * Aug 28, 2013 2299 rferrel purgeExpiredFromArchive now returns the number of files purged. * * * @author rferrel @@ -288,18 +288,22 @@ public class ArchiveConfigManager { * Archive. * * @param archive - * @return the list of expired Files purged + * @return purgeCount */ - public Collection purgeExpiredFromArchive(ArchiveConfig archive) { - Collection filesPurged = new ArrayList(); + public int purgeExpiredFromArchive(ArchiveConfig archive) { String archiveRootDirPath = archive.getRootDir(); File archiveRootDir = new File(archiveRootDirPath); + String[] topLevelDirs = archiveRootDir.list(); + List topLevelDirsNotPurged = new ArrayList(); + int purgeCount = 0; if (topLevelDirs != null) { topLevelDirsNotPurged.addAll(Arrays.asList(topLevelDirs)); + topLevelDirs = null; } + for (CategoryConfig category : archive.getCategoryList()) { Calendar purgeTime = calculateExpiration(archive, category); CategoryFileDateHelper helper = new CategoryFileDateHelper( @@ -323,68 +327,92 @@ public class ArchiveConfigManager { List displayFiles = getDisplayFiles(display, null, purgeTime); for (File file : displayFiles) { - filesPurged.addAll(purgeFile(file, fileDateFilter, - archiveRootDirPath)); + purgeCount += purgeFile(file, fileDateFilter); } } } // check for other expired in top level directories not covered - // by the categories in the archives + // by the categories in the archive. Calendar defaultPurgeTime = calculateExpiration(archive, null); + IOFileFilter fileDateFilter = FileFilterUtils.and(FileFilterUtils + .fileFileFilter(), new FileDateFilter(null, defaultPurgeTime)); for (String topDirName : topLevelDirsNotPurged) { - IOFileFilter fileDateFilter = FileFilterUtils.and(FileFilterUtils - .fileFileFilter(), new FileDateFilter(null, - defaultPurgeTime)); File topLevelDir = new File(archiveRootDir, topDirName); - filesPurged.addAll(purgeFile(topLevelDir, fileDateFilter, - archiveRootDirPath)); + // Keep both top level hidden files and hidden directories. + if (!topLevelDir.isHidden()) { + purgeCount += purgeFile(topLevelDir, fileDateFilter); + } } - return filesPurged; + return purgeCount; } - private Collection purgeFile(File fileToPurge, IOFileFilter filter, - final String archiveRootDir) { - Collection filesPurged = new ArrayList(); + /** + * Recursive method for purging files. Never pass in a directory you do not + * want deleted when purging makes it an empty directory. + * + * @param fileToPurge + * @param filter + * @return purgeCount number of files and directories purged + */ + private int purgeFile(File fileToPurge, IOFileFilter filter) { + int purgeCount = 0; if (fileToPurge.isFile() && filter.accept(fileToPurge)) { if (fileToPurge.delete()) { - filesPurged.add(fileToPurge); + ++purgeCount; + if (statusHandler.isPriorityEnabled(Priority.DEBUG)) { + statusHandler.debug("Purged file: \"" + + fileToPurge.getAbsolutePath() + "\""); + } } else { statusHandler.warn("Failed to purge file: " + fileToPurge.getAbsolutePath()); } - } else if (fileToPurge.isDirectory()) { - Collection expiredFilesInDir = FileUtils.listFiles( - fileToPurge, filter, FileFilterUtils.trueFileFilter()); + } else if (fileToPurge.isDirectory() && !fileToPurge.isHidden()) { + // Purge only visible directories. + File[] expiredFilesInDir = fileToPurge.listFiles(); for (File dirFile : expiredFilesInDir) { - filesPurged.addAll(purgeFile(dirFile, filter, archiveRootDir)); + purgeCount += purgeFile(dirFile, filter); } - // if the directory is empty and not the archive root dir, then - // delete it - if (fileToPurge.list().length == 0 - && !fileToPurge.getAbsolutePath().equals(archiveRootDir)) { + // Attempt to delete empty directory. + if ((purgeCount >= expiredFilesInDir.length) + && (fileToPurge.list().length == 0)) { if (!fileToPurge.delete()) { statusHandler.warn("Failed to purge directory: " + fileToPurge.getAbsolutePath()); + } else { + ++purgeCount; + if (statusHandler.isPriorityEnabled(Priority.DEBUG)) { + statusHandler.debug("Purged directory: \"" + + fileToPurge.getAbsolutePath() + + File.separator + "\""); + } } } } - return filesPurged; + return purgeCount; } + /** + * Get expiration time for the category. + * + * @param archive + * @param category + * @return expireCal + */ private Calendar calculateExpiration(ArchiveConfig archive, CategoryConfig category) { - Calendar newCal = TimeUtil.newGmtCalendar(); + Calendar expireCal = TimeUtil.newGmtCalendar(); int retHours = category == null || category.getRetentionHours() == 0 ? archive .getRetentionHours() : category.getRetentionHours(); if (retHours != 0) { - newCal.add(Calendar.HOUR, (-1) * retHours); + expireCal.add(Calendar.HOUR, (-1) * retHours); } - return newCal; + return expireCal; } /** diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java index 7cd3030a51..6f87b7e73d 100644 --- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java @@ -41,6 +41,7 @@ import com.raytheon.uf.common.time.util.TimeUtil; * ------------ ---------- ----------- -------------------------- * Jun 21, 2013 1965 bgonzale Initial creation * Aug 03, 2013 2224 rferrel Changes for new configuration files. + * Aug 28, 2013 2299 rferrel Changes in IFileDateHelper. * * * @@ -137,10 +138,11 @@ public class CategoryFileDateHelper implements IFileDateHelper { * * @see * com.raytheon.uf.common.archive.config.IFileDateHelper#getFileDate(java - * .lang.String) + * .io.File) */ @Override - public Calendar getFileDate(String filenamePath) { + public Calendar getFileDate(File file) { + String filenamePath = file.getAbsolutePath(); String pathForFilePatternCheck = filenamePath.substring(rootDir .length()); String pathForDirPatternCheck = FilenameUtils @@ -165,7 +167,6 @@ public class CategoryFileDateHelper implements IFileDateHelper { if (timestamp == null) { // no matching pattern, use file last modified date - File file = new File(filenamePath); timestamp = file.lastModified(); } diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/FileDateFilter.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/FileDateFilter.java index 16dc9fd732..d349faf652 100644 --- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/FileDateFilter.java +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/FileDateFilter.java @@ -40,6 +40,7 @@ import com.raytheon.uf.common.time.util.TimeUtil; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jun 18, 2013 1965 bgonzale Initial creation + * Aug 28, 2013 2299 rferrel Reject hidden directories. * * * @@ -79,7 +80,9 @@ public class FileDateFilter implements IOFileFilter { this.end = end; } - /* (non-Javadoc) + /* + * (non-Javadoc) + * * @see org.apache.commons.io.filefilter.IOFileFilter#accept(java.io.File) */ @Override @@ -90,16 +93,16 @@ public class FileDateFilter implements IOFileFilter { return accept(new File(dirName), fileName); } - /* (non-Javadoc) - * @see org.apache.commons.io.filefilter.IOFileFilter#accept(java.io.File, java.lang.String) + /* + * (non-Javadoc) + * + * @see org.apache.commons.io.filefilter.IOFileFilter#accept(java.io.File, + * java.lang.String) */ @Override public boolean accept(File dir, String name) { - String dirPath = dir.getAbsolutePath(); - boolean endsWithSeparator = dirPath.endsWith(File.separator); - dirPath = endsWithSeparator ? dirPath : dirPath + File.separator; - String fileFullPath = dirPath + name; - Calendar fileDate = helper.getFileDate(fileFullPath); + File file = new File(dir, name); + Calendar fileDate = helper.getFileDate(file); boolean isAfterEqualsStart = start == null || fileDate.after(start) || fileDate.equals(start); boolean isBeforeEqualsEnd = end == null || fileDate.before(end) @@ -112,9 +115,8 @@ public class FileDateFilter implements IOFileFilter { */ private static final IFileDateHelper DEFAULT_FILE_DATE_HELPER = new IFileDateHelper() { @Override - public Calendar getFileDate(String filenamePath) { + public Calendar getFileDate(File file) { // use file last modified date - File file = new File(filenamePath); long lastModifiedMillis = file.lastModified(); Calendar result = TimeUtil.newGmtCalendar(); result.setTimeInMillis(lastModifiedMillis); diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/IFileDateHelper.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/IFileDateHelper.java index 16a4f24d0c..368e6fe6d4 100644 --- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/IFileDateHelper.java +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/IFileDateHelper.java @@ -19,6 +19,7 @@ **/ package com.raytheon.uf.common.archive.config; +import java.io.File; import java.util.Calendar; /** @@ -31,6 +32,7 @@ import java.util.Calendar; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jun 21, 2013 bgonzale Initial creation + * Aug 28, 2013 2299 rferrel Change getFileDate argument. * * * @@ -40,6 +42,12 @@ import java.util.Calendar; public interface IFileDateHelper { - public Calendar getFileDate(String filenamePath); + /** + * Get data associated with the file. + * + * @param file + * @return calendar + */ + public Calendar getFileDate(File file); } diff --git a/edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/purge/ArchivePurger.java b/edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/purge/ArchivePurger.java index af69b267eb..7806756290 100644 --- a/edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/purge/ArchivePurger.java +++ b/edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/purge/ArchivePurger.java @@ -19,7 +19,6 @@ **/ package com.raytheon.uf.edex.archive.purge; -import java.io.File; import java.util.Collection; import com.raytheon.uf.common.archive.config.ArchiveConfig; @@ -39,6 +38,8 @@ import com.raytheon.uf.common.status.UFStatus.Priority; * ------------ ---------- ----------- -------------------------- * May 6, 2013 1965 bgonzale Initial creation * Added info logging for purge counts. + * Aug 28, 2013 2299 rferrel manager.purgeExpiredFromArchive now returns + * number of files purged. * * * @@ -57,14 +58,13 @@ public class ArchivePurger { ArchiveConfigManager manager = ArchiveConfigManager.getInstance(); Collection archives = manager.getArchives(); for (ArchiveConfig archive : archives) { - Collection deletedFiles = manager - .purgeExpiredFromArchive(archive); + int purgeCount = manager.purgeExpiredFromArchive(archive); if (statusHandler.isPriorityEnabled(Priority.INFO)) { StringBuilder sb = new StringBuilder(archive.getName()); sb.append("::Archive Purged "); - sb.append(deletedFiles.size()); + sb.append(purgeCount); sb.append(" file"); - if (deletedFiles.size() != 1) { + if (purgeCount != 1) { sb.append("s"); } sb.append("."); diff --git a/tests/unit/com/raytheon/uf/common/archive/ArchiveConfigManagerTest.java b/tests/unit/com/raytheon/uf/common/archive/ArchiveConfigManagerTest.java index f70e3bb98b..1b3eeacd9d 100644 --- a/tests/unit/com/raytheon/uf/common/archive/ArchiveConfigManagerTest.java +++ b/tests/unit/com/raytheon/uf/common/archive/ArchiveConfigManagerTest.java @@ -64,6 +64,7 @@ import com.raytheon.uf.common.util.TestUtil; * May 7, 2013 1965 bgonzale Initial creation. * Added additional test data for file newer than purge * time but in directory that is older than purge time. + * Aug 28, 2013 2299 rferrel purgeExpiredFromArchive now returns number of files purged. * * * @@ -91,11 +92,13 @@ public class ArchiveConfigManagerTest { private final DateFormat mmFormat = new SimpleDateFormat("mm"); - private Collection archiveFiles = new ArrayList(); + private final Collection archiveFiles = new ArrayList(); - private Collection purgeFiles = new ArrayList(); + private final Collection purgeFiles = new ArrayList(); - private Collection archiveSelectedDisplays = new HashSet(); + private final Collection allFiles = new ArrayList(); + + private final Collection archiveSelectedDisplays = new HashSet(); private Calendar referenceCalendar; @@ -115,25 +118,9 @@ public class ArchiveConfigManagerTest { if (referenceCalendar == null) { setupTimes(); } - File testLocalization = TestUtil - .setupTestClassDir(PathManagerFactoryTest.class); PathManagerFactoryTest.initLocalization(); - // after setting up test localization get the production config files - // and copy them to the the test localization directory. - // utility/common_static/base/archive - File prodConfigDir = new File( - "../edexOsgi/com.raytheon.uf.edex.archive/utility"); - Collection configs = FileUtils.listFiles(prodConfigDir, - FileFilterUtils.trueFileFilter(), - FileFilterUtils.trueFileFilter()); - File destDir = new File(testLocalization, - "utility/common_static/base/archive"); - for (File srcConfig : configs) { - FileUtils.copyFileToDirectory(srcConfig, destDir); - } - ArchiveConfigManager manager = ArchiveConfigManager.getInstance(); ArchiveConfig archiveProcessed = manager.getArchive(PROCESSED); @@ -355,6 +342,7 @@ public class ArchiveConfigManagerTest { dir.mkdirs(); resultFile.createNewFile(); + allFiles.add(resultFile); return resultFile; } @@ -414,9 +402,21 @@ public class ArchiveConfigManagerTest { public void testArchiveManagerPurge() throws IOException { ArchiveConfigManager manager = ArchiveConfigManager.getInstance(); Collection filesFoundInPurge = new ArrayList(); + int purgeCount = 0; for (ArchiveConfig a : manager.getArchives()) { - filesFoundInPurge.addAll(manager.purgeExpiredFromArchive(a)); + purgeCount += manager.purgeExpiredFromArchive(a); + } + + // assertEquals( + // + // "The expected number of purged files and number of purge files not the same", + // purgeCount, purgeFiles.size()); + + for (File file : allFiles) { + if (!file.exists()) { + filesFoundInPurge.add(file); + } } assertEquals(