Issue #1965 Commit for purge and archive with tests.
A lot of this ended up being duplication of Roger's code. Roger's changes should go first, and then this commit will be merged and refactored with his code in an amend. Amend: better test coverage, fixes to old other pattern integrated with Roger's changes. added header comments, deleted FileArchiveElement. removed search and filter code. Change-Id: I4e48842a75b9da27062349fee01099e1ea13f59e Former-commit-id: 991622c763767a1e291be8c9f3c10da2b5fd7e32
This commit is contained in:
parent
bd9251eba0
commit
321f153547
13 changed files with 506 additions and 346 deletions
|
@ -5,10 +5,8 @@ Bundle-SymbolicName: com.raytheon.uf.common.archive
|
|||
Bundle-Version: 1.0.0.qualifier
|
||||
Bundle-Vendor: RAYTHEON
|
||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
||||
Export-Package: com.raytheon.uf.common.archive,
|
||||
com.raytheon.uf.common.archive.config,
|
||||
com.raytheon.uf.common.archive.exception,
|
||||
com.raytheon.uf.common.archive.file
|
||||
Export-Package: com.raytheon.uf.common.archive.config,
|
||||
com.raytheon.uf.common.archive.exception
|
||||
Require-Bundle: com.raytheon.uf.common.util;bundle-version="1.12.1174",
|
||||
com.raytheon.uf.common.localization;bundle-version="1.12.1174",
|
||||
com.raytheon.uf.common.status;bundle-version="1.12.1174",
|
||||
|
|
|
@ -1,45 +0,0 @@
|
|||
/**
|
||||
* This software was developed and / or modified by Raytheon Company,
|
||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
||||
*
|
||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
||||
* This software product contains export-restricted data whose
|
||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
||||
* to non-U.S. persons whether in the United States or abroad requires
|
||||
* an export license or other authorization.
|
||||
*
|
||||
* Contractor Name: Raytheon Company
|
||||
* Contractor Address: 6825 Pine Street, Suite 340
|
||||
* Mail Stop B8
|
||||
* Omaha, NE 68106
|
||||
* 402.291.0100
|
||||
*
|
||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
||||
* further licensing information.
|
||||
**/
|
||||
package com.raytheon.uf.common.archive;
|
||||
|
||||
/**
|
||||
* Factory for accessing an archive CategoryManager.
|
||||
*
|
||||
* <pre>
|
||||
*
|
||||
* SOFTWARE HISTORY
|
||||
*
|
||||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* May 06, 2013 1966 bgonzale Initial creation
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* @author bgonzale
|
||||
* @version 1.0
|
||||
*/
|
||||
|
||||
public class ArchiveManagerFactory {
|
||||
|
||||
public static IArchiveManager getManager() {
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
|
@ -1,56 +0,0 @@
|
|||
/**
|
||||
* This software was developed and / or modified by Raytheon Company,
|
||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
||||
*
|
||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
||||
* This software product contains export-restricted data whose
|
||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
||||
* to non-U.S. persons whether in the United States or abroad requires
|
||||
* an export license or other authorization.
|
||||
*
|
||||
* Contractor Name: Raytheon Company
|
||||
* Contractor Address: 6825 Pine Street, Suite 340
|
||||
* Mail Stop B8
|
||||
* Omaha, NE 68106
|
||||
* 402.291.0100
|
||||
*
|
||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
||||
* further licensing information.
|
||||
**/
|
||||
package com.raytheon.uf.common.archive;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Interface that defines a Archive.
|
||||
*
|
||||
* <pre>
|
||||
*
|
||||
* SOFTWARE HISTORY
|
||||
*
|
||||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* May 7, 2013 1966 bgonzale Initial creation
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* @author bgonzale
|
||||
* @version 1.0
|
||||
*/
|
||||
|
||||
public interface IArchive {
|
||||
|
||||
/**
|
||||
* Obtain a list archive elements that have expired.
|
||||
*
|
||||
* @param archiveName
|
||||
* - Name of Archive Data.
|
||||
* @param categoryNameList
|
||||
* - List of categories to check. All categories are checked if
|
||||
* the list is null or empty.
|
||||
* @return archiveElementList
|
||||
*/
|
||||
List<IArchiveElement> getExpiredElements(String archiveName,
|
||||
List<String> categoryNameList);
|
||||
|
||||
}
|
|
@ -1,53 +0,0 @@
|
|||
/**
|
||||
* This software was developed and / or modified by Raytheon Company,
|
||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
||||
*
|
||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
||||
* This software product contains export-restricted data whose
|
||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
||||
* to non-U.S. persons whether in the United States or abroad requires
|
||||
* an export license or other authorization.
|
||||
*
|
||||
* Contractor Name: Raytheon Company
|
||||
* Contractor Address: 6825 Pine Street, Suite 340
|
||||
* Mail Stop B8
|
||||
* Omaha, NE 68106
|
||||
* 402.291.0100
|
||||
*
|
||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
||||
* further licensing information.
|
||||
**/
|
||||
package com.raytheon.uf.common.archive;
|
||||
|
||||
import com.raytheon.uf.common.archive.exception.ArchiveException;
|
||||
|
||||
/**
|
||||
* Defines an Archive Element.
|
||||
*
|
||||
* <pre>
|
||||
*
|
||||
* SOFTWARE HISTORY
|
||||
*
|
||||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* May 7, 2013 1966 bgonzale Initial creation
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* @author bgonzale
|
||||
* @version 1.0
|
||||
*/
|
||||
|
||||
public interface IArchiveElement {
|
||||
|
||||
/**
|
||||
* Purge this element from the archive.
|
||||
*
|
||||
* @return true if successful; false otherwise.
|
||||
* @throws ArchiveException
|
||||
*/
|
||||
public boolean purge() throws ArchiveException;
|
||||
|
||||
public String getName();
|
||||
|
||||
}
|
|
@ -1,45 +0,0 @@
|
|||
/**
|
||||
* This software was developed and / or modified by Raytheon Company,
|
||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
||||
*
|
||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
||||
* This software product contains export-restricted data whose
|
||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
||||
* to non-U.S. persons whether in the United States or abroad requires
|
||||
* an export license or other authorization.
|
||||
*
|
||||
* Contractor Name: Raytheon Company
|
||||
* Contractor Address: 6825 Pine Street, Suite 340
|
||||
* Mail Stop B8
|
||||
* Omaha, NE 68106
|
||||
* 402.291.0100
|
||||
*
|
||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
||||
* further licensing information.
|
||||
**/
|
||||
package com.raytheon.uf.common.archive;
|
||||
|
||||
/**
|
||||
* Interface definition for Archive access.
|
||||
*
|
||||
* <pre>
|
||||
*
|
||||
* SOFTWARE HISTORY
|
||||
*
|
||||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* May 06, 2013 1966 bgonzale Initial creation
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* @author bgonzale
|
||||
* @version 1.0
|
||||
*/
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface IArchiveManager {
|
||||
|
||||
public List<IArchive> getArchives();
|
||||
|
||||
}
|
|
@ -54,7 +54,7 @@ import javax.xml.bind.annotation.XmlRootElement;
|
|||
*
|
||||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* May 1, 2013 1966 rferrel Initial creation
|
||||
* May 1, 2013 1966 rferrel Initial creation
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
|
@ -193,4 +193,5 @@ public class ArchiveConfig implements Comparable<ArchiveConfig> {
|
|||
sb.append("]");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -29,6 +29,7 @@ import java.text.MessageFormat;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
@ -39,10 +40,13 @@ import java.util.regex.Pattern;
|
|||
|
||||
import javax.xml.bind.JAXB;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.commons.io.filefilter.FileFilterUtils;
|
||||
import org.apache.commons.io.filefilter.IOFileFilter;
|
||||
import org.apache.commons.io.filefilter.RegexFileFilter;
|
||||
|
||||
import com.raytheon.uf.common.archive.exception.ArchiveException;
|
||||
import com.raytheon.uf.common.localization.IPathManager;
|
||||
import com.raytheon.uf.common.localization.LocalizationFile;
|
||||
import com.raytheon.uf.common.localization.PathManagerFactory;
|
||||
|
@ -62,7 +66,8 @@ import com.raytheon.uf.common.util.FileUtil;
|
|||
*
|
||||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* May 1, 2013 1966 rferrel Initial creation
|
||||
* May 1, 2013 1966 rferrel Initial creation
|
||||
* May 29, 2013 1965 bgonzale Added archive creation method and purge.
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
|
@ -161,6 +166,148 @@ public class ArchiveConfigManager {
|
|||
return unmarshalArhiveConfigFromXmlFile(lFile);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the Collection of Archives.
|
||||
*/
|
||||
public Collection<ArchiveConfig> getArchives() {
|
||||
return archiveMap.values();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an archive in the given archive directory from the Files filtered
|
||||
* by the given Archive out of the ArchiveElements.
|
||||
*
|
||||
* @param archive
|
||||
* Archive configuration to create Archive from
|
||||
* @param category
|
||||
* Archive Category configuration
|
||||
* @param archiveDir
|
||||
* Destination directory of the Archive
|
||||
* @param displaysSelectedForArchive
|
||||
* List of display elements to filter inclusion into Archive
|
||||
* @param start
|
||||
* Start time boundary of the Archive
|
||||
* @param end
|
||||
* End time boundary of the Archive
|
||||
* @return the list of files added to the created archive.
|
||||
* @throws IOException
|
||||
* @throws ArchiveException
|
||||
*/
|
||||
public Collection<File> createArchive(ArchiveConfig archive,
|
||||
CategoryConfig category, File archiveDir,
|
||||
Collection<String> displaysSelectedForArchive, Calendar start,
|
||||
Calendar end) throws IOException, ArchiveException {
|
||||
Collection<File> archivedFiles = null;
|
||||
if (archiveDir.exists() || archiveDir.mkdirs()) {
|
||||
Collection<File> filesToArchive = new ArrayList<File>();
|
||||
|
||||
for (String displayLabel : displaysSelectedForArchive) {
|
||||
File[] files = getDisplayFiles(archive.getName(),
|
||||
category.getName(), displayLabel, start, end);
|
||||
filesToArchive.addAll(Arrays.asList(files));
|
||||
}
|
||||
|
||||
String rootDirString = archive.getRootDir();
|
||||
String archiveDirString = archiveDir.getAbsolutePath();
|
||||
|
||||
archivedFiles = new ArrayList<File>(filesToArchive.size());
|
||||
rootDirString = (rootDirString.endsWith(File.separator) ? rootDirString
|
||||
.substring(0, rootDirString.lastIndexOf(File.separatorChar))
|
||||
: rootDirString);
|
||||
for (File srcFile : filesToArchive) {
|
||||
String fileAbsPath = srcFile.getAbsolutePath();
|
||||
File newArchiveDir = getDirRelativeToArchiveDirFromRoot(
|
||||
srcFile.isDirectory(), fileAbsPath, rootDirString,
|
||||
archiveDirString);
|
||||
FileUtils.forceMkdir(newArchiveDir);
|
||||
|
||||
if (srcFile.isDirectory()) {
|
||||
FileUtils.copyDirectory(srcFile, newArchiveDir);
|
||||
archivedFiles.addAll(Arrays.asList(newArchiveDir
|
||||
.listFiles()));
|
||||
} else {
|
||||
FileUtils.copyFileToDirectory(srcFile, newArchiveDir);
|
||||
String filename = FilenameUtils.getName(fileAbsPath);
|
||||
File dstFile = new File(newArchiveDir, filename);
|
||||
archivedFiles.add(dstFile);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
StringBuilder sbuff = new StringBuilder(
|
||||
"Failed to create archive for Archive: ");
|
||||
sbuff.append(archive);
|
||||
sbuff.append(" and Category: ");
|
||||
sbuff.append(category);
|
||||
throw new ArchiveException(sbuff.toString());
|
||||
}
|
||||
return archivedFiles;
|
||||
}
|
||||
|
||||
private File getDirRelativeToArchiveDirFromRoot(boolean fileIsDir,
|
||||
String fileAbsPath,
|
||||
String rootDirString, String archiveDirString) {
|
||||
String path = null;
|
||||
if (fileIsDir) {
|
||||
path = fileAbsPath;
|
||||
} else {
|
||||
path = FilenameUtils.getFullPath(fileAbsPath);
|
||||
}
|
||||
String newArchivePathString = path.replace(rootDirString,
|
||||
archiveDirString);
|
||||
return new File(newArchivePathString);
|
||||
}
|
||||
|
||||
/**
|
||||
* Purge the Files that fall outside of the time frame constraints for the
|
||||
* Archive.
|
||||
*
|
||||
* @param archive
|
||||
* @return the list of expired Files purged
|
||||
*/
|
||||
public Collection<File> purgeExpiredFromArchive(ArchiveConfig archive) {
|
||||
Collection<File> filesPurged = new ArrayList<File>();
|
||||
|
||||
for (CategoryConfig category : archive.getCategoryList()) {
|
||||
Calendar purgeTime = calculateExpiration(archive, category);
|
||||
|
||||
for (String displayLabel : getDisplayLabels(archive.getName(),
|
||||
category.getName())) {
|
||||
File[] displayFiles = getDisplayFiles(archive.getName(),
|
||||
category.getName(), displayLabel, null, purgeTime);
|
||||
for (File file : displayFiles) {
|
||||
if (file.isFile()) {
|
||||
filesPurged.add(file);
|
||||
} else if (file.isDirectory()) {
|
||||
filesPurged.addAll(FileUtils.listFiles(file,
|
||||
FileFilterUtils.fileFileFilter(),
|
||||
FileFilterUtils.trueFileFilter()));
|
||||
}
|
||||
FileUtils.deleteQuietly(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
return filesPurged;
|
||||
}
|
||||
|
||||
private Calendar calculateExpiration(ArchiveConfig archive,
|
||||
CategoryConfig category) {
|
||||
Calendar newCal = TimeUtil.newGmtCalendar();
|
||||
int retHours = category.getRetentionHours() == 0 ? archive
|
||||
.getRetentionHours() : category.getRetentionHours();
|
||||
if (retHours != 0) {
|
||||
newCal.add(Calendar.HOUR, (-1) * retHours);
|
||||
}
|
||||
return newCal;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the archive with the given name; return null if it does not
|
||||
* exist.
|
||||
*/
|
||||
public ArchiveConfig getArchive(String archiveName) {
|
||||
return archiveMap.get(archiveName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the Archive Configuration data to the localized file.
|
||||
*
|
||||
|
|
|
@ -1,101 +0,0 @@
|
|||
/**
|
||||
* This software was developed and / or modified by Raytheon Company,
|
||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
||||
*
|
||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
||||
* This software product contains export-restricted data whose
|
||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
||||
* to non-U.S. persons whether in the United States or abroad requires
|
||||
* an export license or other authorization.
|
||||
*
|
||||
* Contractor Name: Raytheon Company
|
||||
* Contractor Address: 6825 Pine Street, Suite 340
|
||||
* Mail Stop B8
|
||||
* Omaha, NE 68106
|
||||
* 402.291.0100
|
||||
*
|
||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
||||
* further licensing information.
|
||||
**/
|
||||
package com.raytheon.uf.common.archive.file;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import com.raytheon.uf.common.archive.IArchiveElement;
|
||||
import com.raytheon.uf.common.archive.exception.ArchiveException;
|
||||
|
||||
/**
|
||||
* Archive element for a file or directory.
|
||||
*
|
||||
* <pre>
|
||||
*
|
||||
* SOFTWARE HISTORY
|
||||
*
|
||||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* May 13, 2013 1966 rferrel Initial creation
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* @author rferrel
|
||||
* @version 1.0
|
||||
*/
|
||||
|
||||
public class FileArchiveElement implements IArchiveElement {
|
||||
|
||||
private String name;
|
||||
|
||||
public FileArchiveElement(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
*
|
||||
* @see com.raytheon.uf.common.archive.IArchiveElement#purge()
|
||||
*/
|
||||
@Override
|
||||
public boolean purge() throws ArchiveException {
|
||||
File file = new File(name);
|
||||
|
||||
boolean state = false;
|
||||
try {
|
||||
if (!file.exists()) {
|
||||
state = true;
|
||||
} else {
|
||||
if (file.isDirectory()) {
|
||||
deleteContents(file);
|
||||
}
|
||||
state = file.delete();
|
||||
}
|
||||
} catch (SecurityException ex) {
|
||||
throw new ArchiveException(ex.getMessage());
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
*
|
||||
* @see com.raytheon.uf.common.archive.IArchiveElement#getName()
|
||||
*/
|
||||
@Override
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete all files and sub-directories.
|
||||
*
|
||||
* @param directory
|
||||
*/
|
||||
private void deleteContents(File directory) {
|
||||
File[] files = directory.listFiles();
|
||||
for (File file : files) {
|
||||
if (file.isDirectory()) {
|
||||
deleteContents(file);
|
||||
}
|
||||
file.delete();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -6,4 +6,4 @@ Bundle-Version: 1.0.0.qualifier
|
|||
Bundle-Vendor: RAYTHEON
|
||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
|
||||
Export-Package: com.raytheon.uf.edex.archive.purge
|
||||
Import-Package: com.raytheon.uf.common.archive
|
||||
Import-Package: com.raytheon.uf.common.archive.config
|
||||
|
|
|
@ -19,31 +19,21 @@
|
|||
**/
|
||||
package com.raytheon.uf.edex.archive.purge;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Collection;
|
||||
|
||||
import com.raytheon.uf.common.archive.ArchiveManagerFactory;
|
||||
import com.raytheon.uf.common.archive.IArchive;
|
||||
import com.raytheon.uf.common.archive.IArchiveElement;
|
||||
import com.raytheon.uf.common.archive.IArchiveManager;
|
||||
import com.raytheon.uf.common.archive.exception.ArchiveException;
|
||||
import com.raytheon.uf.common.status.IUFStatusHandler;
|
||||
import com.raytheon.uf.common.status.UFStatus;
|
||||
import com.raytheon.uf.common.archive.config.ArchiveConfig;
|
||||
import com.raytheon.uf.common.archive.config.ArchiveConfigManager;
|
||||
|
||||
/**
|
||||
* Purge task to purge archived data based on configured expiration.
|
||||
*
|
||||
* TODO This is a purge that will run on a timer scheduled in spring properties
|
||||
* that is not related to the expiration date of data. Another possible solution
|
||||
* would be to programatically schedule a timer when an archive is created based
|
||||
* on the expiration date of the archive.
|
||||
*
|
||||
* <pre>
|
||||
*
|
||||
* SOFTWARE HISTORY
|
||||
*
|
||||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* May 6, 2013 bgonzale Initial creation
|
||||
* May 6, 2013 1965 bgonzale Initial creation
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
|
@ -52,34 +42,15 @@ import com.raytheon.uf.common.status.UFStatus;
|
|||
*/
|
||||
|
||||
public class ArchivePurger {
|
||||
private static final IUFStatusHandler logger = UFStatus
|
||||
.getHandler(ArchivePurger.class);
|
||||
|
||||
/**
|
||||
* Purge expired elements from the archives.
|
||||
*/
|
||||
public void purge() {
|
||||
IArchiveManager manager = ArchiveManagerFactory.getManager();
|
||||
List<IArchive> archives = manager.getArchives();
|
||||
for (IArchive archive : archives) {
|
||||
// TODO fix...
|
||||
for (IArchiveElement element : archive.getExpiredElements(null,
|
||||
null)) {
|
||||
try {
|
||||
if (!element.purge()) {
|
||||
String elementName = element == null ? "null" : element
|
||||
.getName();
|
||||
logger.error("ArchivePurger unable to purge element "
|
||||
+ elementName);
|
||||
}
|
||||
} catch (ArchiveException e) {
|
||||
String elementName = element == null ? "null" : element
|
||||
.getName();
|
||||
logger.error("ArchivePurger unable to purge element "
|
||||
+ elementName, e);
|
||||
}
|
||||
}
|
||||
public static void purge() {
|
||||
ArchiveConfigManager manager = ArchiveConfigManager.getInstance();
|
||||
Collection<ArchiveConfig> archives = manager.getArchives();
|
||||
for (ArchiveConfig archive : archives) {
|
||||
manager.purgeExpiredFromArchive(archive);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -50,6 +50,7 @@
|
|||
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.raytheon.uf.edex.database"/>
|
||||
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.mchange"/>
|
||||
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/org.apache.commons.management"/>
|
||||
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/org.apache.commons.io"/>
|
||||
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.facebook.thrift"/>
|
||||
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.raytheon.uf.common.datastorage"/>
|
||||
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.raytheon.uf.common.event"/>
|
||||
|
@ -75,7 +76,6 @@
|
|||
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.raytheon.edex.plugin.redbook"/>
|
||||
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.raytheon.uf.edex.plugin.level"/>
|
||||
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.raytheon.uf.common.dataplugin.level"/>
|
||||
<classpathentry combineaccessrules="false" exported="true" kind="src" path="/com.raytheon.uf.common.archive"/>
|
||||
<classpathentry kind="src" path="/com.raytheon.uf.common.dataplugin.ffmp"/>
|
||||
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.common.dataaccess"/>
|
||||
<classpathentry kind="src" path="/com.raytheon.uf.common.monitor"/>
|
||||
|
@ -93,5 +93,6 @@
|
|||
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.common.datadelivery.harvester"/>
|
||||
<classpathentry combineaccessrules="false" kind="src" path="/javax.jms"/>
|
||||
<classpathentry combineaccessrules="false" kind="src" path="/org.apache.commons.cxf"/>
|
||||
<classpathentry kind="src" path="/com.raytheon.uf.common.archive"/>
|
||||
<classpathentry kind="output" path="bin"/>
|
||||
</classpath>
|
||||
|
|
|
@ -0,0 +1,339 @@
|
|||
/**
|
||||
* This software was developed and / or modified by Raytheon Company,
|
||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
||||
*
|
||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
||||
* This software product contains export-restricted data whose
|
||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
||||
* to non-U.S. persons whether in the United States or abroad requires
|
||||
* an export license or other authorization.
|
||||
*
|
||||
* Contractor Name: Raytheon Company
|
||||
* Contractor Address: 6825 Pine Street, Suite 340
|
||||
* Mail Stop B8
|
||||
* Omaha, NE 68106
|
||||
* 402.291.0100
|
||||
*
|
||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
||||
* further licensing information.
|
||||
**/
|
||||
package com.raytheon.uf.common.archive;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.text.DateFormat;
|
||||
import java.text.FieldPosition;
|
||||
import java.text.MessageFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.commons.io.filefilter.FileFilterUtils;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.raytheon.uf.common.archive.config.ArchiveConfig;
|
||||
import com.raytheon.uf.common.archive.config.ArchiveConfigManager;
|
||||
import com.raytheon.uf.common.archive.config.CategoryConfig;
|
||||
import com.raytheon.uf.common.archive.exception.ArchiveException;
|
||||
import com.raytheon.uf.common.localization.PathManagerFactoryTest;
|
||||
import com.raytheon.uf.common.time.util.TimeUtil;
|
||||
import com.raytheon.uf.common.util.FileUtil;
|
||||
import com.raytheon.uf.common.util.TestUtil;
|
||||
|
||||
/**
|
||||
* Test ArchiveConfigManager Archive Ingest Purge and Archive Creation.
|
||||
*
|
||||
* <pre>
|
||||
*
|
||||
* SOFTWARE HISTORY
|
||||
*
|
||||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* May 7, 2013 1965 bgonzale Initial creation
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* @author bgonzale
|
||||
* @version 1.0
|
||||
*/
|
||||
|
||||
public class ArchiveConfigManagerTest {
|
||||
|
||||
private static final String RAW = "Raw";
|
||||
|
||||
private static final String SAT_CAT_NAME = "Satellite";
|
||||
|
||||
private static File TEST_DIR = TestUtil
|
||||
.setupTestClassDir(ArchiveConfigManagerTest.class);
|
||||
|
||||
private final DateFormat yyyyMMFormat = new SimpleDateFormat("yyyyMM");
|
||||
|
||||
private final DateFormat ddFormat = new SimpleDateFormat("dd");
|
||||
|
||||
private final DateFormat hhFormat = new SimpleDateFormat("HH");
|
||||
|
||||
private final DateFormat mmFormat = new SimpleDateFormat("mm");
|
||||
|
||||
private ArchiveConfigManager manager;
|
||||
|
||||
private ArchiveConfig archive;
|
||||
|
||||
private Collection<File> archiveFiles = new ArrayList<File>();
|
||||
|
||||
private Collection<File> expiredFiles = new ArrayList<File>();
|
||||
|
||||
private Collection<File> purgeFiles = new ArrayList<File>();
|
||||
|
||||
private Calendar referenceCalendar;
|
||||
|
||||
private Calendar archiveStart;
|
||||
|
||||
private Calendar archiveEnd;
|
||||
|
||||
private List<String> selectedForArchive;
|
||||
|
||||
private File archiveDir;
|
||||
|
||||
private static String TEST_ARCHIVE_DIR = "testArchiveDir";
|
||||
|
||||
/**
|
||||
* @throws java.lang.Exception
|
||||
*/
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
if (referenceCalendar == null) {
|
||||
setupTimes();
|
||||
}
|
||||
File testLocalization = TestUtil
|
||||
.setupTestClassDir(PathManagerFactoryTest.class);
|
||||
|
||||
PathManagerFactoryTest.initLocalization();
|
||||
|
||||
// after setting up test localization get the production config files
|
||||
// and copy them to the the test localization directory.
|
||||
// utility/common_static/base/archive
|
||||
File prodConfigDir = new File(
|
||||
"../edexOsgi/com.raytheon.uf.edex.archive/utility");
|
||||
Collection<File> configs = FileUtils.listFiles(prodConfigDir,
|
||||
FileFilterUtils.trueFileFilter(),
|
||||
FileFilterUtils.trueFileFilter());
|
||||
File destDir = new File(testLocalization,
|
||||
"utility/common_static/base/archive");
|
||||
for (File srcConfig : configs) {
|
||||
FileUtils.copyFileToDirectory(srcConfig, destDir);
|
||||
}
|
||||
|
||||
manager = ArchiveConfigManager.getInstance();
|
||||
archive = manager.getArchive(RAW);
|
||||
|
||||
// configure the test archive to use the test data dir
|
||||
archive.setRootDir(TEST_DIR.getAbsolutePath() + "/data_store/");
|
||||
|
||||
// MessageFormat keys
|
||||
// {0} yyyyMM {1} dd {2} hh {3} mm
|
||||
|
||||
// **** grib1 ****
|
||||
MessageFormat grib1Format = new MessageFormat(
|
||||
"/grib/{0}{1}/18/NWS_160/GRID255/{2}{3}Z_F001_APCP-ZETA98_KSTR_{1}{2}{3}_125544891.grib.{0}{1}{2}");
|
||||
CategoryConfig grib1Cat = getCategory(archive, "Model grib");
|
||||
createTestFiles(grib1Format, getRetentionHours(archive, grib1Cat),
|
||||
false, archiveStart, archiveEnd);
|
||||
// manager is not configured internally until this is called...
|
||||
manager.getDisplayLabels(archive.getName(), grib1Cat.getName());
|
||||
|
||||
// **** sat ****
|
||||
CategoryConfig satCat = getCategory(archive, SAT_CAT_NAME);
|
||||
MessageFormat satFormat = new MessageFormat(
|
||||
"/sat/{0}{1}/{2}/GOES-13/{2}{3}Z_SOUND-VIS_10km_EAST-CONUS-TIGE59_KNES_128453.satz.{0}{1}{2}");
|
||||
createTestFiles(satFormat, getRetentionHours(archive, satCat), true,
|
||||
archiveStart, archiveEnd);
|
||||
// manager is not configured internally until this is called...
|
||||
manager.getDisplayLabels(archive.getName(), satCat.getName());
|
||||
|
||||
// **** acars ****
|
||||
CategoryConfig otherCat = getCategory(archive, "Model other");
|
||||
int otherCatRetentionHours = getRetentionHours(archive, otherCat);
|
||||
MessageFormat acarsFormat = new MessageFormat(
|
||||
"/acars/{0}{1}/{2}/IUAB01_CWAO_{1}{2}{3}_22714956.bufr.{0}{1}{2}");
|
||||
createTestFiles(acarsFormat, otherCatRetentionHours, false,
|
||||
archiveStart, archiveEnd);
|
||||
|
||||
// **** binlightning ****
|
||||
MessageFormat binlightningFormat = new MessageFormat(
|
||||
"/binlightning/{0}{1}/{2}/SFUS41_KWBC_{1}{2}{3}_22725485.nldn.{0}{1}{2}");
|
||||
createTestFiles(binlightningFormat, otherCatRetentionHours, false,
|
||||
archiveStart, archiveEnd);
|
||||
|
||||
// **** bufrsigwx ****
|
||||
MessageFormat bufrsigwxFormat = new MessageFormat(
|
||||
"/bufrsigwx/{0}{1}/{2}/JUWE96_KKCI_{1}{2}{3}_31368878.bufr.{0}{1}{2}");
|
||||
createTestFiles(bufrsigwxFormat, otherCatRetentionHours, false,
|
||||
archiveStart, archiveEnd);
|
||||
// manager is not configured internally until this is called...
|
||||
manager.getDisplayLabels(archive.getName(), otherCat.getName());
|
||||
|
||||
// create test archive data dir
|
||||
archiveDir = new File(TEST_DIR, TEST_ARCHIVE_DIR);
|
||||
// create archive elements for selected list
|
||||
selectedForArchive = new ArrayList<String>();
|
||||
selectedForArchive.add("GOES-13");
|
||||
|
||||
}
|
||||
|
||||
private int getRetentionHours(ArchiveConfig archive, CategoryConfig category) {
|
||||
return category.getRetentionHours() == 0 ? archive.getRetentionHours()
|
||||
: category.getRetentionHours();
|
||||
}
|
||||
|
||||
private CategoryConfig getCategory(ArchiveConfig archive,
|
||||
String categoryName) {
|
||||
CategoryConfig category = null;
|
||||
for (CategoryConfig c : archive.getCategoryList()) {
|
||||
if (c.getName().equals(categoryName)) {
|
||||
category = c;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return category;
|
||||
}
|
||||
|
||||
private void createTestFiles(MessageFormat dataFileNameFormat,
|
||||
int retentionHours, boolean isSelected, Calendar start, Calendar end)
|
||||
throws IOException {
|
||||
// create data file newer than purge time, within archive time, and
|
||||
// isSelected
|
||||
File dataFile = createDataFile(end, dataFileNameFormat);
|
||||
if (isSelected) {
|
||||
archiveFiles.add(dataFile);
|
||||
}
|
||||
|
||||
// create data file newer than purge time, within archive time, but not
|
||||
// in selected
|
||||
Calendar moreThanOneDayOld = (Calendar) referenceCalendar.clone();
|
||||
moreThanOneDayOld.add(Calendar.DAY_OF_MONTH, -1);
|
||||
createDataFile(moreThanOneDayOld, dataFileNameFormat);
|
||||
|
||||
// create data file older than purge time
|
||||
Calendar lessThanExpiredCalendar = (Calendar) referenceCalendar
|
||||
.clone();
|
||||
lessThanExpiredCalendar.add(Calendar.HOUR, (-1
|
||||
* retentionHours - 1));
|
||||
dataFile = createDataFile(lessThanExpiredCalendar,
|
||||
dataFileNameFormat);
|
||||
expiredFiles.add(dataFile);
|
||||
purgeFiles.add(dataFile);
|
||||
|
||||
// create data file newer than purge time and outside of archive time
|
||||
// frame
|
||||
Calendar newerThanArchiveEnd = (Calendar) end.clone();
|
||||
newerThanArchiveEnd.add(Calendar.HOUR, 3);
|
||||
createDataFile(newerThanArchiveEnd, dataFileNameFormat);
|
||||
}
|
||||
|
||||
private void setupTimes() {
|
||||
referenceCalendar = TimeUtil.newGmtCalendar();
|
||||
referenceCalendar.set(Calendar.MINUTE, 0);
|
||||
referenceCalendar.set(Calendar.SECOND, 0);
|
||||
|
||||
archiveStart = (Calendar) referenceCalendar.clone();
|
||||
archiveEnd = (Calendar) referenceCalendar.clone();
|
||||
|
||||
archiveStart.add(Calendar.HOUR, -20);
|
||||
archiveEnd.add(Calendar.HOUR, -3);
|
||||
|
||||
yyyyMMFormat.setCalendar(referenceCalendar);
|
||||
ddFormat.setCalendar(referenceCalendar);
|
||||
hhFormat.setCalendar(referenceCalendar);
|
||||
mmFormat.setCalendar(referenceCalendar);
|
||||
}
|
||||
|
||||
private File createDataFile(Calendar referenceCalendar,
|
||||
MessageFormat fileFormat) throws IOException {
|
||||
Date referenceTime = referenceCalendar.getTime();
|
||||
|
||||
String yyyyMM = yyyyMMFormat.format(referenceTime);
|
||||
String dd = ddFormat.format(referenceTime);
|
||||
String hh = hhFormat.format(referenceTime);
|
||||
String mm = mmFormat.format(referenceTime);
|
||||
String[] formatArgs = new String[] { yyyyMM, dd, hh, mm };
|
||||
|
||||
String filename = fileFormat.format(formatArgs, new StringBuffer(),
|
||||
new FieldPosition(0)).toString();
|
||||
File resultFile = new File(archive.getRootDir(), filename);
|
||||
String dirname = FilenameUtils
|
||||
.getFullPath(resultFile.getAbsolutePath());
|
||||
File dir = new File(dirname);
|
||||
|
||||
dir.mkdirs();
|
||||
resultFile.createNewFile();
|
||||
return resultFile;
|
||||
}
|
||||
|
||||
private Collection<String> createFileNameListNoRootDir(File rootDir,
|
||||
Collection<File> files) {
|
||||
List<String> result = new ArrayList<String>(files.size());
|
||||
for (File f : files) {
|
||||
String absPath = f.getAbsolutePath();
|
||||
String fileRelativePath = absPath.replace(
|
||||
rootDir.getAbsolutePath(), "");
|
||||
result.add(fileRelativePath);
|
||||
}
|
||||
Collections.sort(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws java.lang.Exception
|
||||
*/
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
FileUtil.deleteDir(TEST_DIR);
|
||||
FileUtil.deleteDir(archiveDir);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testArchiveManagerCreateArchive() throws IOException,
|
||||
ArchiveException {
|
||||
CategoryConfig satCategory = getCategory(archive, SAT_CAT_NAME);
|
||||
Collection<File> archivedFiles = manager.createArchive(archive,
|
||||
satCategory, archiveDir, selectedForArchive, archiveStart,
|
||||
archiveEnd);
|
||||
assertEquals(
|
||||
"The expected archive files and the archived files are not the same",
|
||||
createFileNameListNoRootDir(new File(archive.getRootDir()),
|
||||
archiveFiles),
|
||||
createFileNameListNoRootDir(archiveDir, archivedFiles));
|
||||
// check archive directory for files.
|
||||
Collection<File> filesFoundInArchive = FileUtils.listFiles(archiveDir,
|
||||
FileFilterUtils.trueFileFilter(),
|
||||
FileFilterUtils.trueFileFilter());
|
||||
assertEquals(
|
||||
"The expected archive files in the files found in the archive are not the same",
|
||||
archivedFiles, filesFoundInArchive);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testArchiveManagerPurge() throws IOException {
|
||||
Collection<File> filesFoundInPurge = manager
|
||||
.purgeExpiredFromArchive(archive);
|
||||
// sort for comparison
|
||||
List<File> purgeFilesList = new ArrayList<File>(purgeFiles);
|
||||
Collections.sort(purgeFilesList);
|
||||
List<File> foundFilesList = new ArrayList<File>(filesFoundInPurge);
|
||||
Collections.sort(foundFilesList);
|
||||
assertEquals(
|
||||
"The expected purge files and the files purged are not the same",
|
||||
purgeFilesList, foundFilesList);
|
||||
}
|
||||
|
||||
}
|
|
@ -43,6 +43,7 @@ import com.raytheon.uf.edex.registry.ebxml.services.query.adhoc.AdhocQueryExpres
|
|||
* ------------ ---------- ----------- --------------------------
|
||||
* Feb 06, 2013 1543 djohnson Moved out of SerializationUtilTest.
|
||||
* Apr 17, 2013 1914 djohnson Add W3CEndpointReference.
|
||||
* May 29, 2013 1965 bgonzale Added archive classes.
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
|
@ -61,6 +62,8 @@ public class TestJaxbableClassesLocator implements IJaxbableClassesLocator {
|
|||
private static final List JAXB_CLASSES;
|
||||
static {
|
||||
Class<?>[] array = new Class<?>[] {
|
||||
com.raytheon.uf.common.archive.config.ArchiveConfig.class,
|
||||
com.raytheon.uf.common.archive.config.CategoryConfig.class,
|
||||
com.raytheon.edex.plugin.redbook.decoder.RedbookFcstMap.class,
|
||||
com.raytheon.uf.common.datadelivery.registry.AdhocSubscription.class,
|
||||
com.raytheon.uf.common.datadelivery.registry.Connection.class,
|
||||
|
|
Loading…
Add table
Reference in a new issue