xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #1966 Changes to CategoryConfig to make parsing more efficent.

Browse source 
Change-Id: I3fc9bf03eccea8ae63cb37a4469e512c1fac9e87

Former-commit-id: 00ed24f081 [formerly 24f1c70168] [formerly 365af6b7ce] [formerly 365af6b7ce [formerly cbee731447]] [formerly 9263b54845 [formerly 365af6b7ce [formerly cbee731447] [formerly 9263b54845 [formerly d8a510b87d026d57eb282583c4cf2f786749dc95]]]]
Former-commit-id: 9263b54845
Former-commit-id: c9d911bbb8ab90460a3b74ae86b587cc4b8858b4 [formerly e443885c0974b7dc457a2ea698d05b926b780848] [formerly 838eacc802 [formerly fce5c2ca1e]]
Former-commit-id: 838eacc802
Former-commit-id: 618feeda6d
This commit is contained in:
Roger Ferrel 2013-05-23 14:08:59 -05:00
parent f4c6efa78e
commit 589be27351
12 changed files with 605 additions and 198 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/data/FileInfo.java
View file

@ -32,7 +32,8 @@ import org.eclipse.core.runtime.jobs.Job;
import com.raytheon.uf.common.util.FileUtil;
/**
* This class uses a obtains information on a File in a Job in order to remove from the UI thread.
* This class uses a obtains information on a File in a Job in order to remove
* from the UI thread.
*
* <pre>
*

2
cots/org.apache.commons.io/.classpath
View file

@ -2,7 +2,7 @@
<classpath>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
<classpathentry exported="true" kind="lib" path="commons-io-2.4.jar"/>
<classpathentry exported="true" kind="lib" path="commons-io-2.4.jar" sourcepath="commons-io-2.4-sources.jar"/>
<classpathentry exported="true" kind="lib" path="commons-io-2.4-javadoc.jar"/>
<classpathentry exported="true" kind="lib" path="commons-io-2.4-sources.jar"/>
<classpathentry kind="output" path="bin"/>

5
cots/org.apache.commons.io/META-INF/MANIFEST.MF
View file

@ -3,9 +3,7 @@ Bundle-ManifestVersion: 2
Bundle-Name: Io
Bundle-SymbolicName: org.apache.commons.io
Bundle-Version: 2.4
Bundle-ClassPath: commons-io-2.4.jar,
commons-io-2.4-javadoc.jar,
commons-io-2.4-sources.jar
Bundle-ClassPath: commons-io-2.4.jar
Bundle-Vendor: Apache
Export-Package: org.apache.commons.io,
org.apache.commons.io.comparator,
@ -14,3 +12,4 @@ Export-Package: org.apache.commons.io,
org.apache.commons.io.monitor,
org.apache.commons.io.output
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Bundle-ActivationPolicy: lazy

4
cots/org.apache.commons.io/build.properties
View file

@ -1,4 +1,2 @@
bin.includes = META-INF/,\
commons-io-2.4.jar,\
commons-io-2.4-javadoc.jar,\
commons-io-2.4-sources.jar
commons-io-2.4.jar

8
edexOsgi/com.raytheon.uf.common.archive/.classpath
View file

@ -1,11 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="src" path="/com.raytheon.uf.common.localization"/>
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.common.status"/>
<classpathentry combineaccessrules="false" kind="src" path="/org.junit"/>
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.viz.core"/>
<classpathentry combineaccessrules="false" kind="src" path="/com.raytheon.uf.common.util"/>
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
<classpathentry kind="src" path="src"/>
<classpathentry kind="output" path="bin"/>
</classpath>

4
edexOsgi/com.raytheon.uf.common.archive/META-INF/MANIFEST.MF
View file

@ -11,4 +11,6 @@ Export-Package: com.raytheon.uf.common.archive,
com.raytheon.uf.common.archive.file
Require-Bundle: com.raytheon.uf.common.util;bundle-version="1.12.1174",
com.raytheon.uf.common.localization;bundle-version="1.12.1174",
com.raytheon.uf.common.status;bundle-version="1.12.1174"
com.raytheon.uf.common.status;bundle-version="1.12.1174",
org.apache.commons.io;bundle-version="2.4.0",
com.raytheon.uf.common.time;bundle-version="1.12.1174"

7
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfig.java
View file

@ -27,8 +27,6 @@ import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.junit.Assert;
/**
* Archive data information for retention, purging and archiving. An example:
*
@ -115,7 +113,6 @@ public class ArchiveConfig implements Comparable<ArchiveConfig> {
* @param name
*/
public void setName(String name) {
Assert.assertNotNull(name);
this.name = name;
}
@ -135,12 +132,11 @@ public class ArchiveConfig implements Comparable<ArchiveConfig> {
* @param rootDir
*/
public void setRootDir(String rootDir) {
Assert.assertNotNull(rootDir);
this.rootDir = rootDir;
}
/**
* Retrive the archive's retention hours.
* Retrieve the archive's retention hours.
*
* @return retentionHours
*/
@ -154,7 +150,6 @@ public class ArchiveConfig implements Comparable<ArchiveConfig> {
* @param retentionHours
*/
public void setRetentionHours(int retentionHours) {
Assert.assertTrue(retentionHours > 0);
this.retentionHours = retentionHours;
}

485
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java
View file

@ -24,27 +24,33 @@ import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.text.FieldPosition;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.bind.JAXB;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.RegexFileFilter;
import com.raytheon.uf.common.localization.IPathManager;
import com.raytheon.uf.common.localization.LocalizationContext;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationLevel;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType;
import com.raytheon.uf.common.localization.LocalizationFile;
import com.raytheon.uf.common.localization.PathManagerFactory;
import com.raytheon.uf.common.localization.exception.LocalizationException;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.common.util.FileUtil;
/**
@ -64,39 +70,33 @@ import com.raytheon.uf.common.util.FileUtil;
* @version 1.0
*/
public class ArchiveConfigManager {
private final static ArchiveConfigManager instance = new ArchiveConfigManager();
private final IUFStatusHandler statusHandler = UFStatus
.getHandler(ArchiveConfigManager.class);
private final static ArchiveConfigManager instance = new ArchiveConfigManager();
public final String ARCHIVE_DIR = "archive";
private IPathManager pathMgr;
private final Map<String, ArchiveConfig> archiveMap = new HashMap<String, ArchiveConfig>();
private DirectoryFilter directoryFilter = new DirectoryFilter();
private Pattern subPatterns[] = new Pattern[] { Pattern.compile("\\1"),
Pattern.compile("\\2"), Pattern.compile("\\3"),
Pattern.compile("\\4"), Pattern.compile("\\5"),
Pattern.compile("\\6"), Pattern.compile("\\7"),
Pattern.compile("\\8"), Pattern.compile("\\9"), };
private Pattern yearPattern = Pattern.compile("$\\{YYYY}");
private Pattern monthPattern = Pattern.compile("$\\{MM}");
private Pattern dayPattern = Pattern.compile("$\\{DD}");
private Pattern hourPattern = Pattern.compile("$\\{HH}");
private Map<ArchiveConfig, Map<CategoryConfig, Map<String, List<File>>>> displayDirMap = new HashMap<ArchiveConfig, Map<CategoryConfig, Map<String, List<File>>>>();
public final static ArchiveConfigManager getInstance() {
return instance;
}
/**
* Private constructor for singleton.
*/
private ArchiveConfigManager() {
pathMgr = PathManagerFactory.getPathManager();
try {
reset();
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e);
}
}
/**
@ -105,51 +105,33 @@ public class ArchiveConfigManager {
* @return archiveConfigList
*/
private LocalizationFile[] getArchiveConfigFiles() {
Map<String, LocalizationFile> confFileMap = new HashMap<String, LocalizationFile>();
LocalizationContext ctx = pathMgr.getContext(
LocalizationType.COMMON_STATIC, LocalizationLevel.BASE);
LocalizationFile[] files = pathMgr.listFiles(ctx, ARCHIVE_DIR,
LocalizationFile[] files = pathMgr.listStaticFiles(ARCHIVE_DIR,
new String[] { ".xml" }, false, true);
for (LocalizationFile lf : files) {
confFileMap.put(lf.getName().trim(), lf);
}
ctx = pathMgr.getContext(LocalizationType.COMMON_STATIC,
LocalizationLevel.SITE);
files = pathMgr.listFiles(ctx, ARCHIVE_DIR, new String[] { ".xml" },
false, true);
for (LocalizationFile lf : files) {
confFileMap.put(lf.getName().trim(), lf);
}
files = new LocalizationFile[confFileMap.size()];
int i = 0;
for (String key : confFileMap.keySet()) {
files[i] = confFileMap.get(key);
++i;
}
return files;
}
public String[] getArchiveDataNamesList() throws IOException,
LocalizationException {
if (archiveMap.size() == 0) {
LocalizationFile[] files = getArchiveConfigFiles();
for (LocalizationFile lFile : files) {
ArchiveConfig archiveConfig = unmarshalArhiveConfigFromXmlFile(lFile);
archiveMap.put(archiveConfig.getName(), archiveConfig);
}
}
/**
* Get a sorted list of the archive data names.
*
* @return names
*/
public String[] getArchiveDataNamesList() {
String[] names = archiveMap.keySet().toArray(new String[0]);
Arrays.sort(names, 0, names.length, String.CASE_INSENSITIVE_ORDER);
return names;
}
/**
* Obtain the names of the categories for the named archive data.
*
* @param archiveConfigName
* @return categoryNames
*/
public String[] getCategoryNames(String archiveConfigName) {
return getCategoryNames(archiveMap.get(archiveConfigName));
}
/**
* @param archiveConfig
* @return
@ -166,38 +148,286 @@ public class ArchiveConfigManager {
return names;
}
/**
* Load the archiveConfig information form the localized file.
*
* @param lFile
* @return archiveConfig
* @throws IOException
* @throws LocalizationException
*/
public ArchiveConfig loadArchiveData(LocalizationFile lFile)
throws IOException, LocalizationException {
return unmarshalArhiveConfigFromXmlFile(lFile);
}
/**
* Save the Archive Configuration data to the localized file.
*
* @param lFile
* @param archiveConfig
* @throws IOException
* @throws LocalizationException
*/
public void saveArchiveConfig(LocalizationFile lFile,
ArchiveConfig archiveConfig) throws IOException,
LocalizationException {
marshalArchiveConfigToXmlFile(archiveConfig, lFile);
ArchiveConfig archiveConfig) {
try {
marshalArchiveConfigToXmlFile(archiveConfig, lFile);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e);
}
}
private void marshalArchiveConfigToXmlFile(ArchiveConfig archiveConfig,
LocalizationFile lFile) throws IOException, LocalizationException {
OutputStream stream = null;
try {
stream = lFile.openOutputStream();
JAXB.marshal(archiveConfig, stream);
} finally {
if (stream != null) {
try {
stream.close();
} catch (IOException ex) {
// ignore
/**
* Add the file entry to the desired display map.
*
* @param file
* @param archiveConfig
* @param categoryConfig
* @param displayLabel
*/
private void addDirToDisplayMap(File file, ArchiveConfig archiveConfig,
CategoryConfig categoryConfig, String displayLabel) {
Map<CategoryConfig, Map<String, List<File>>> cMap = displayDirMap
.get(archiveConfig);
if (cMap == null) {
cMap = new HashMap<CategoryConfig, Map<String, List<File>>>();
displayDirMap.put(archiveConfig, cMap);
}
Map<String, List<File>> dirMap = cMap.get(categoryConfig);
if (dirMap == null) {
dirMap = new HashMap<String, List<File>>();
cMap.put(categoryConfig, dirMap);
}
List<File> fileList = dirMap.get(displayLabel);
if (fileList == null) {
fileList = new ArrayList<File>();
dirMap.put(displayLabel, fileList);
}
fileList.add(file);
}
/**
* Get a list of directories associated with the display label.
*
* @param archiveConfig
* @param categoryConfig
* @param displayLabel
* @return
*/
private File[] getDirFromDisplayMap(ArchiveConfig archiveConfig,
CategoryConfig categoryConfig, String displayLabel) {
Map<CategoryConfig, Map<String, List<File>>> cMap = displayDirMap
.get(archiveConfig);
File[] result = new File[0];
if (cMap == null) {
return result;
}
Map<String, List<File>> dirMap = cMap.get(categoryConfig);
if (dirMap == null) {
return result;
}
List<File> fileList = dirMap.get(displayLabel);
if (fileList == null) {
return result;
}
return fileList.toArray(result);
}
/**
* Clear maps and reloads the maps from the configuration information.
*
* @throws IOException
* @throws LocalizationException
*/
public void reset() throws IOException, LocalizationException {
clearDisplayMap();
archiveMap.clear();
LocalizationFile[] files = getArchiveConfigFiles();
for (LocalizationFile lFile : files) {
ArchiveConfig archiveConfig = unmarshalArhiveConfigFromXmlFile(lFile);
archiveMap.put(archiveConfig.getName(), archiveConfig);
}
}
/**
* Walk the display directory maps clearing all entries.
*/
private void clearDisplayMap() {
for (ArchiveConfig archiveConfig : displayDirMap.keySet()) {
Map<CategoryConfig, Map<String, List<File>>> cMap = displayDirMap
.get(archiveConfig);
for (CategoryConfig categoryConfig : cMap.keySet()) {
Map<String, List<File>> dirMap = cMap.get(categoryConfig);
dirMap.remove(categoryConfig);
for (String displayLabel : dirMap.keySet()) {
List<File> fileList = dirMap.get(displayLabel);
fileList.clear();
}
dirMap.clear();
}
}
}
/**
* Get a list of directories/files for the desired display label bounded by
* the start and end time inclusive.
*
* @param archiveName
* - name of the ArchiveConfig containing the category
* @param categoryName
* - name of the CategoryConfig containing the display
* @param displayLabel
* - the display label
* @param startCal
* - Start time if null epoch time is used
* @param endCal
* - End time if null current simulated time is used.
* @return files
*/
public File[] getDisplayFiles(String archiveName, String categoryName,
List<String> dispalyList, Calendar startTime, Calendar endTime) {
// TODO implement
return null;
String displayLabel, Calendar startCal, Calendar endCal) {
long startTime = 0L;
if (startCal != null) {
// Set to beginning of the hour
Calendar cal = (Calendar) startCal.clone();
cal.set(Calendar.MILLISECOND, 0);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
startTime = cal.getTimeInMillis();
}
Calendar cal = null;
if (endCal != null) {
cal = (Calendar) endCal.clone();
} else {
cal = TimeUtil.newCalendar();
}
// Set to start of the next hour.
cal.set(Calendar.MILLISECOND, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MINUTE, 0);
cal.add(Calendar.HOUR_OF_DAY, 1);
long endTime = cal.getTimeInMillis();
return getDisplayFiles(archiveName, categoryName, displayLabel,
startTime, endTime);
}
/**
* Get all of the files/directories associated with the display label.
*
* @param archiveName
* @param categoryName
* @param displayLabel
* @return files
*/
public File[] getDisplayFiles(String archiveName, String categoryName,
String displayLabel) {
return getDisplayFiles(archiveName, categoryName, displayLabel, null,
null);
}
/**
* Get the files/directories associated with the display label with time
* stamps between inclusive start time and exclusive end time.
*
* @param archiveName
* @param categoryName
* @param displayLabel
* @param startTime
* @param endTime
* @return files
*/
private File[] getDisplayFiles(String archiveName, String categoryName,
String displayLabel, long startTime, long endTime) {
ArchiveConfig archiveConfig = archiveMap.get(archiveName);
CategoryConfig categoryConfig = findCategory(archiveConfig,
categoryName);
String[] indexValues = categoryConfig.getDateGroupIndices().split(
"\\s*,\\s*");
int yearIndex = Integer.parseInt(indexValues[0]);
int monthIndex = Integer.parseInt(indexValues[1]);
int dayIndex = Integer.parseInt(indexValues[2]);
int hourIndex = Integer.parseInt(indexValues[3]);
String filePatternStr = categoryConfig.getFilePattern();
boolean dirOnly = (filePatternStr == null)
|| ".*".equals(filePatternStr);
File dirs[] = getDirFromDisplayMap(archiveConfig, categoryConfig,
displayLabel);
int beginIndex = archiveConfig.getRootDir().length();
Calendar fileCal = TimeUtil.newCalendar();
fileCal.setTimeZone(TimeZone.getTimeZone("UTC"));
List<File> fileList = new ArrayList<File>();
if (dirOnly) {
Pattern pattern = Pattern.compile(categoryConfig.getDirPattern());
for (File dir : dirs) {
String path = dir.getAbsolutePath().substring(beginIndex);
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
int year = Integer.parseInt(matcher.group(yearIndex));
// Adjust month value to Calendar's 0 - 11
int month = Integer.parseInt(matcher.group(monthIndex)) - 1;
int day = Integer.parseInt(matcher.group(dayIndex));
int hour = Integer.parseInt(matcher.group(hourIndex));
fileCal.set(year, month, day, hour, 0, 0);
long fileTime = fileCal.getTimeInMillis();
if ((startTime <= fileTime) && (fileTime < endTime)) {
fileList.add(dir);
}
}
}
} else {
Pattern pattern = Pattern.compile(categoryConfig.getDirPattern()
+ File.separator + categoryConfig.getFilePattern());
final Pattern filePattern = Pattern.compile("^" + filePatternStr
+ "$");
for (File dir : dirs) {
List<File> fList = FileUtil.listDirFiles(dir, new FileFilter() {
@Override
public boolean accept(File pathname) {
return filePattern.matcher(pathname.getName())
.matches();
}
}, false);
for (File file : fList) {
String path = file.getAbsolutePath().substring(beginIndex);
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
int year = Integer.parseInt(matcher.group(yearIndex));
// Adjust month value to Calendar's 0 - 11
int month = Integer.parseInt(matcher.group(monthIndex)) - 1;
int day = Integer.parseInt(matcher.group(dayIndex));
int hour = Integer.parseInt(matcher.group(hourIndex));
fileCal.set(year, month, day, hour, 0, 0);
long fileTime = fileCal.getTimeInMillis();
if ((startTime <= fileTime) && (fileTime < endTime)) {
fileList.add(dir);
}
}
}
}
}
File[] files = fileList.toArray(new File[0]);
return files;
}
/**
@ -211,43 +441,67 @@ public class ArchiveConfigManager {
*/
public String[] getDisplayLabels(String archiveName, String categoryName) {
ArchiveConfig archiveConfig = archiveMap.get(archiveName);
CategoryConfig category = findCategory(archiveConfig, categoryName);
CategoryConfig categoryConfig = findCategory(archiveConfig,
categoryName);
String dirPattern = categoryConfig.getDirPattern();
File rootFile = new File(archiveConfig.getRootDir());
List<File> dirList = findDirs(rootFile);
String[] subExpr = dirPattern.split(File.separator);
List<File> dirs = new ArrayList<File>();
dirs.add(rootFile);
List<File> tmpDirs = new ArrayList<File>();
List<File> swpDirs = null;
Pattern pattern = Pattern.compile(category.getDirPattern());
for (String regex : subExpr) {
Pattern subPattern = Pattern.compile("^" + regex + "$");
IOFileFilter filter = FileFilterUtils
.makeDirectoryOnly(new RegexFileFilter(subPattern));
for (File dir : dirs) {
File[] list = dir.listFiles();
if (list != null) {
List<File> dirList = Arrays.asList(list);
tmpDirs.addAll(Arrays.asList(FileFilterUtils.filter(filter,
dirList)));
}
}
swpDirs = dirs;
dirs = tmpDirs;
tmpDirs = swpDirs;
tmpDirs.clear();
}
// index for making directory paths' relative to the root path.
int beginIndex = rootFile.getAbsolutePath().length() + 1;
Pattern pattern = Pattern.compile("^" + dirPattern + "$");
TreeSet<String> displays = new TreeSet<String>(
String.CASE_INSENSITIVE_ORDER);
MessageFormat msgfmt = new MessageFormat(categoryConfig.getDisplay());
StringBuffer sb = new StringBuffer();
for (File dir : dirList) {
Matcher matcher = pattern.matcher(dir.getAbsolutePath());
if (matcher.matches()) {
String display = category.getDisplay();
int groupCnt = matcher.groupCount();
FieldPosition pos0 = new FieldPosition(0);
for (int i = 0; i < groupCnt; ++i) {
Matcher subMatcher = subPatterns[i].matcher(display);
sb.setLength(0);
while (subMatcher.find()) {
subMatcher.appendReplacement(sb, matcher.group(i));
}
subMatcher.appendTail(sb);
display = sb.toString();
for (File file : dirs) {
String path = file.getAbsolutePath().substring(beginIndex);
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
sb.setLength(0);
String[] args = new String[matcher.groupCount() + 1];
args[0] = matcher.group();
for (int i = 1; i < args.length; ++i) {
args[i] = matcher.group(i);
}
displays.add(display);
String displayLabel = msgfmt.format(args, sb, pos0).toString();
addDirToDisplayMap(file, archiveConfig, categoryConfig,
displayLabel);
displays.add(displayLabel);
}
}
return displays.toArray(new String[0]);
}
private List<File> findDirs(File parentDir) {
return FileUtil.listDirFiles(parentDir, directoryFilter, true);
}
/**
* Obtain the category with the given name.
*
@ -265,6 +519,33 @@ public class ArchiveConfigManager {
return null;
}
/**
* This does the work of saving an instance of ArchiveConfig to a localized
* file.
*
* @param archiveConfig
* @param lFile
* @throws IOException
* @throws LocalizationException
*/
private void marshalArchiveConfigToXmlFile(ArchiveConfig archiveConfig,
LocalizationFile lFile) throws IOException, LocalizationException {
OutputStream stream = null;
stream = lFile.openOutputStream();
JAXB.marshal(archiveConfig, stream);
stream.close();
lFile.save();
}
/**
* This does the work of taking a localized file and returning a instance of
* the ArchiveConfig class from the data in the file.
*
* @param lFile
* @return archiveConfig
* @throws IOException
* @throws LocalizationException
*/
private ArchiveConfig unmarshalArhiveConfigFromXmlFile(
LocalizationFile lFile) throws IOException, LocalizationException {
ArchiveConfig archiveConfig = null;
@ -283,20 +564,4 @@ public class ArchiveConfigManager {
}
return archiveConfig;
}
private class DirectoryFilter implements FileFilter {
/*
* (non-Javadoc)
*
* @see java.io.FileFilter#accept(java.io.File)
*/
@Override
public boolean accept(File pathname) {
if (pathname.isDirectory() && !pathname.getName().startsWith(".")) {
return true;
}
return false;
}
}
}

78
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java
View file

@ -24,8 +24,6 @@ import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.junit.Assert;
/**
* Information on a category for a given archive.
*
@ -36,10 +34,10 @@ import org.junit.Assert;
* &lt;name>redbook&lt;/name>
* &lt;!-- When 0 default to the parent archive's retentionHours -->
* &lt;retentionHours>0&lt;/retentionHours>
* &lt;dirPattern>hdf5/redbook/([^/]*)/&lt;/dirPattern>
* &lt;display>redbook - \1&lt;/display>
* &lt;saveDir>hdf5/redbook/([^/]*)/&lt;/saveDir>
* &lt;saveFiles>redbook-${YYYY}-${MM}-${DD}-${HH}\..*&lt;/saveFiles>
* &lt;dirPattern>hdf5/(redbook)&lt;/dirPattern>
* &lt;display>{1}&lt;/display>
* &lt;filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*&lt;/filePattern>
* &lt;dateGroupIndices>2,3,4,5&lt;/dateGroupIndices>
* &lt;/category>
* </pre>
*
@ -49,11 +47,9 @@ import org.junit.Assert;
* &lt;category>
* &lt;name>Model grib&lt;/name>
* &lt;retentionHours>0&lt;/retentionHours>
* &lt;dirPattern>grib/[^/]*&#47[^/]*&#47(.*\)&lt;/dirPattern>
* &lt;display>\1&lt;/display>
* &lt;saveDir>grib/${YYYY}${MM}${DD}/${HH}/(.*)&lt;/saveDir>
* &lt;saveFiles>.*&lt;/saveFiles>
* &lt;selectList>grib/${YYYY}${MM}${DD}/${HH}/NCEP_QPF/&lt;/selectList>
* &lt;dirPattern>grib/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)&lt;/dirPattern>
* &lt;display>{5}&lt;/display>
* &lt;dateGroupIndices>1,2,3,4&lt;/dateGroupIndices>
* &lt;/category>
* </pre>
*
@ -93,7 +89,7 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
* example:
*
* <pre>
* &lt;dirPattern>grib2/[^/]*&#47;[^/]*&#47;(.*)/&lt;/dirPattern>
* &lt;dirPattern>grib2/\d{8}/\d{2}/(.*)/&lt;/dirPattern>
* </pre>
*/
@XmlElement(name = "dirPattern")
@ -104,26 +100,33 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
* dirPatern may be displayed. For example:
*
* <pre>
* &lt;dirName>grib2/[^/]*&#47;[^/]*&#47;(.*)&lt;/dirName>
* &lt;display>grib2 - \1&lt;/display>
* &lt;dirName>(grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)&lt;/dirName>
* &lt;display>{1} - {6}&lt;/display>
* </pre>
*
* The \1 will be replaced with directory names 2 levels down from the grib2
* directory.
* The {1} will be replaced by the first group (grib2) in the regex
* expression in dirName. The {6} is the sixth group (.*). {0} is the whole
* expression match.
*/
@XmlElement(name = "display")
private String display;
/**
* A pattern to find the directories to save. This may contain the year,
* month, day and hour information. For example:
* A comma separated list of 4 numbers representing the group indices
* specifying the location of the numeric date time stamp. The indices must
* be in the order of year, month, day and hour. The group numbering starts
* with the first group in the dirPattern and continues with any grouping in
* the filePattern.
*
* <pre>
* &lt;saveDir>grib2/${YYYY}${MM}{$DD}/${HH}/.*&lt;/saveDir>
* &lt;dirPattern>hdf5/(redbook)&lt;/dirPattern>
* &lt;display>{1}&lt;/display>
* &lt;filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*&lt;/filePattern>
* &lt;dateGroupIndices>2,3,4,5&lt;/dateGroupIndices>
* </pre>
*/
@XmlElement(name = "saveDir")
private String saveDir;
@XmlElement(name = "dateGroupIndices")
private String dateGroupIndices;
/**
* A saveDir directory may contain files with data for several days. This
@ -135,8 +138,8 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
* &lt;saveFile>redbook-${YYYY}-${MM}-${DD}-${HH}\..*&lt;saveFiles>
* </pre>
*/
@XmlElement(name = "saveFiles")
private String saveFiles;
@XmlElement(name = "filePattern")
private String filePattern;
/*
* Constructor.
@ -158,7 +161,6 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
* @param name
*/
public void setName(String name) {
Assert.assertNotNull(name);
this.name = name;
}
@ -175,7 +177,6 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
* Set the retention hours must be greater then zero.
*/
public void setRetentionHours(int retentionHours) {
Assert.assertTrue(retentionHours > 0);
this.retentionHours = retentionHours;
}
@ -194,7 +195,6 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
* @param dirPattern
*/
public void setDirPattern(String dirPattern) {
Assert.assertNotNull(dirPattern);
this.dirPattern = dirPattern;
}
@ -208,22 +208,21 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
}
/**
* Set the disaplay pattern.
* Set the display pattern.
*
* @param display
*/
public void setDisplay(String display) {
Assert.assertNotNull(display);
this.display = display;
}
/**
* Get the save directory pattern..
*
* @return savedir
* @return dateGroups
*/
public String getSaveDir() {
return saveDir;
public String getDateGroupIndices() {
return dateGroupIndices;
}
/**
@ -231,9 +230,8 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
*
* @param saveDir
*/
public void setSaveDir(String saveDir) {
Assert.assertNotNull(saveDir);
this.saveDir = saveDir;
public void setDateGroupIndices(String dateGroupIndices) {
this.dateGroupIndices = dateGroupIndices;
}
/**
@ -241,8 +239,8 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
*
* @return saveFiles
*/
public String getSaveFiles() {
return saveFiles;
public String getFilePattern() {
return filePattern;
}
/**
@ -250,8 +248,8 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
*
* @param saveFiles
*/
public void setSaveFiles(String saveFiles) {
this.saveFiles = saveFiles;
public void setFilePattern(String filePattern) {
this.filePattern = filePattern;
}
/*
@ -275,9 +273,9 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
sb.append("Category [ name: ").append(getName());
sb.append(", retentionHours: ").append(getRetentionHours());
sb.append(", dirPattern: ").append(getDirPattern());
sb.append(", filePattern: ").append(getFilePattern());
sb.append(", display: ").append(getDisplay());
sb.append(", saveDir: ").append(getSaveDir());
sb.append(", saveFiles: ").append(getSaveFiles());
sb.append(", dateGroupIndices: ").append(getDateGroupIndices());
sb.append("]");
return sb.toString();
}

3
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/file/FileArchiveElement.java
View file

@ -21,8 +21,6 @@ package com.raytheon.uf.common.archive.file;
import java.io.File;
import org.junit.Assert;
import com.raytheon.uf.common.archive.IArchiveElement;
import com.raytheon.uf.common.archive.exception.ArchiveException;
@ -48,7 +46,6 @@ public class FileArchiveElement implements IArchiveElement {
private String name;
public FileArchiveElement(String name) {
Assert.assertNotNull(name);
this.name = name;
}

96
edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archive/PROCESSED_DATA.xml
View file

@ -1,4 +1,84 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<!--
This_software_was_developed_and_/_or_modified_by_Raytheon_Company,
pursuant_to_Contract_DG133W-05-CQ-1067_with_the_US_Government.
U.S._EXPORT_CONTROLLED_TECHNICAL_DATA
This_software_product_contains_export-restricted_data_whose
export/transfer/disclosure_is_restricted_by_U.S._law._Dissemination
to_non-U.S._persons_whether_in_the_United_States_or_abroad_requires
an_export_license_or_other_authorization.
Contractor_Name:________Raytheon_Company
Contractor_Address:_____6825_Pine_Street,_Suite_340
________________________Mail_Stop_B8
________________________Omaha,_NE_68106
________________________402.291.0100
See_the_AWIPS_II_Master_Rights_File_("Master_Rights_File.pdf")_for
further_licensing_information.
-->
<!--
The <archive> contains five types of tags:
<name> - Required tag. The id for the archive such as Raw or Processed.
Used in the GUIs to filter which archive to display.
<rootDir> - Required tag. The root directory on the edex server for the archive.
<retentionHours> - Required tag. The default number of hour to retain data in the <rootDir>
<category> - Detail information for display information in the archive GUIs and
overriding the default retention hours. There can be several of these tags.
The <category> has six types of tags:
<name> - Required tag. The id for the category such as grib2 or redbook.
Used in the GUIs to filter what is the display in the table.
<retentionHours> - Optional tag. The retentionHours for this category.
Default is the archive's <retentionHours>.
<dirPattern> - Required tag. A pattern for finding directories for this category.
The pattern is relative to the archive's <rootDir>. Wildcard patterns do not cross directory
delimiter /. Thus to match 3 levels of directories you would need .*/.*/.* .
See patterns and groups section.
<filePattern> - Optional tag. A pattern to find files in the directories that match the <dirPattern>.
Default is everything in the directories that match <dirPattern>.
See patterns and groups section.
<display> - Required tag. The label to display for directories that match <dirPattern>. Any group in the
<dirPattern> may be made part of the label by placing the group's index inside parenthesis, {1}.
More then one directory may match the <dirPattern>. The archive GUIs may clapse them into a single
table entry.
<dateGroupIndicies> - Required tag. A comma separated list of 4 numbers which are the index of the groups in
<dirPattern> and/or <filePattern> that contain in order the year, mouth, day and hour information.
This is used to determine what files/directories to retain or a range of directories/files to copy
for case creation. Note to get the group's index the <dirPattern> and <filePattern> are combined.
Thus if there are 5 groups in the <dirPattern> then the first group in the <filePattern> is index 6.
Patterns and groups.
The <dirPattern> and <filePattern> use Java regex expressions; similar to the ldm's pqact.conf file.
For more details see http://docs.oracle.com/javase/tutorial/essential/regex/
The groupings index start at one. The groups in the <dirPattern> can be used in the <display>. For example:
<dirPattern>(grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<display>{1} - {6}</display>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
Here the <dirPattern> contains six groups. The first group is the literal grib2 which matches only a directory named grib2
that is a sub-directory of the <rootDir>. The groups 2, 3 and 4 break apart the next level of sub-directories into a 4 digit
and two 2 digit groups. This is the expected year, month, day sub-subdirectory indicated by the first 3 entries in
<dateGroupIndices>. The next sub-directory contains the fifth group which is a two digit number representing the hour.
Finally the sixth group will match any sub-directory that in the hour directory.
Thus the directory paths <rootPath>/grib2/20130527/18/GFS will generate the display string, grib2 - GFS, and from the grouping we
can find the year, 201; month, 05; day, 27 and hour, 18.
Example with <filePattern>:
<dirPattern>hdf5/(redbook)</dirPattern>
<display>{1}</display>
<filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
Here the display will only show, redbook. The directory looked at will be <rootPath>/redbook/. The <dateGroupIndices> all
come from the <filePattern> since there is one group in the <dirPattern> the groups in the <filePattern> start at two. This
matches file names redbook-YYYY-MM-DD-HH.<extension>. Thus the file name redbook-2013-05-28-00.hd5 would match the <filePattern>.
NOTE group {0} is a string that matches the whole <dirPattern>. If this is used in the <display> would see every directory that
matches the pattern.
-->
<archive>
<name>Processed</name>
<rootDir>/awips2/edex/data/archive/</rootDir>
@ -6,17 +86,17 @@
<category>
<name>redbook</name>
<retentionHours>0</retentionHours>
<dirPattern>hdf5/redbook/([^/]*)/</dirPattern>
<display>redbook - \1</display>
<saveDir>hdf5/redbook/([^/]*)/</saveDir>
<saveFiles>redbook-${YYYY}-${MM}-${DD}-${HH}\..*</saveFiles>
<dirPattern>hdf5/(redbook)</dirPattern>
<display>{1}</display>
<filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</category>
<category>
<name>obs</name>
<retentionHours>0</retentionHours>
<dirPattern>hdf5/(obs)/</dirPattern>
<display>\1</display>
<saveDir>hdf5/(obs)/</saveDir>
<saveFiles>metar-${YYYY}-${MM}-${DD}-${HH}\.*</saveFiles>
<dirPattern>hdf5/(obs)</dirPattern>
<display>{1}</display>
<filePattern>metar-(\d{4})-(\d{2})-(\d{2})-(\d{2})\.*</filePattern>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</category>
</archive>

108
edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archive/RAW_DATA.xml
View file

@ -1,4 +1,84 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<!--
This_software_was_developed_and_/_or_modified_by_Raytheon_Company,
pursuant_to_Contract_DG133W-05-CQ-1067_with_the_US_Government.
U.S._EXPORT_CONTROLLED_TECHNICAL_DATA
This_software_product_contains_export-restricted_data_whose
export/transfer/disclosure_is_restricted_by_U.S._law._Dissemination
to_non-U.S._persons_whether_in_the_United_States_or_abroad_requires
an_export_license_or_other_authorization.
Contractor_Name:________Raytheon_Company
Contractor_Address:_____6825_Pine_Street,_Suite_340
________________________Mail_Stop_B8
________________________Omaha,_NE_68106
________________________402.291.0100
See_the_AWIPS_II_Master_Rights_File_("Master_Rights_File.pdf")_for
further_licensing_information.
-->
<!--
The <archive> contains five types of tags:
<name> - Required tag. The id for the archive such as Raw or Processed.
Used in the GUIs to filter which archive to display.
<rootDir> - Required tag. The root directory on the edex server for the archive.
<retentionHours> - Required tag. The default number of hour to retain data in the <rootDir>
<category> - Detail information for display information in the archive GUIs and
overriding the default retention hours. There can be several of these tags.
The <category> has six types of tags:
<name> - Required tag. The id for the category such as grib2 or redbook.
Used in the GUIs to filter what is the display in the table.
<retentionHours> - Optional tag. The retentionHours for this category.
Default is the archive's <retentionHours>.
<dirPattern> - Required tag. A pattern for finding directories for this category.
The pattern is relative to the archive's <rootDir>. Wildcard patterns do not cross directory
delimiter /. Thus to match 3 levels of directories you would need .*/.*/.* .
See patterns and groups section.
<filePattern> - Optional tag. A pattern to find files in the directories that match the <dirPattern>.
Default is everything in the directories that match <dirPattern>.
See patterns and groups section.
<display> - Required tag. The label to display for directories that match <dirPattern>. Any group in the
<dirPattern> may be made part of the label by placing the group's index inside parenthesis, {1}.
More then one directory may match the <dirPattern>. The archive GUIs may clapse them into a single
table entry.
<dateGroupIndicies> - Required tag. A comma separated list of 4 numbers which are the index of the groups in
<dirPattern> and/or <filePattern> that contain in order the year, mouth, day and hour information.
This is used to determine what files/directories to retain or a range of directories/files to copy
for case creation. Note to get the group's index the <dirPattern> and <filePattern> are combined.
Thus if there are 5 groups in the <dirPattern> then the first group in the <filePattern> is index 6.
Patterns and groups.
The <dirPattern> and <filePattern> use Java regex expressions; similar to the ldm's pqact.conf file.
For more details see http://docs.oracle.com/javase/tutorial/essential/regex/
The groupings index start at one. The groups in the <dirPattern> can be used in the <display>. For example:
<dirPattern>(grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<display>{1} - {6}</display>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
Here the <dirPattern> contains six groups. The first group is the literal grib2 which matches only a directory named grib2
that is a sub-directory of the <rootDir>. The groups 2, 3 and 4 break apart the next level of sub-directories into a 4 digit
and two 2 digit groups. This is the expected year, month, day sub-subdirectory indicated by the first 3 entries in
<dateGroupIndices>. The next sub-directory contains the fifth group which is a two digit number representing the hour.
Finally the sixth group will match any sub-directory that in the hour directory.
Thus the directory paths <rootPath>/grib2/20130527/18/GFS will generate the display string, grib2 - GFS, and from the grouping we
can find the year, 201; month, 05; day, 27 and hour, 18.
Example with <filePattern>:
<dirPattern>hdf5/(redbook)</dirPattern>
<display>{1}</display>
<filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
Here the display will only show, redbook. The directory looked at will be <rootPath>/redbook/. The <dateGroupIndices> all
come from the <filePattern> since there is one group in the <dirPattern> the groups in the <filePattern> start at two. This
matches file names redbook-YYYY-MM-DD-HH.<extension>. Thus the file name redbook-2013-05-28-00.hd5 would match the <filePattern>.
NOTE group {0} is a string that matches the whole <dirPattern>. If this is used in the <display> would see every directory that
matches the pattern.
-->
<archive>
<name>Raw</name>
<rootDir>/data_store/</rootDir>
@ -6,33 +86,29 @@
<category>
<name>Model grib</name>
<retentionHours>0</retentionHours>
<dirPattern>grib/[^/]*/[^/]*/(.*\)</dirPattern>
<display>\1</display>
<saveDir>grib/${YYYY}${MM}${DD}/${HH}/(.*)</saveDir>
<saveFiles>.*</saveFiles>
<dirPattern>grib/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<display>{5}</display>
<dateGroupIndices>1,2,3,4</dateGroupIndices>
</category>
<category>
<name>Model grib2</name>
<retentionHours>0</retentionHours>
<dirPattern>grib2/[^/]*/[^/]*/(.*\)</dirPattern>
<display>\1</display>
<saveDir>grib2/${YYYY}${MM}${DD}/${HH}/.*</saveDir>
<saveFiles>.*</saveFiles>
<dirPattern>(grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<display>{1} - {6}</display>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</category>
<category>
<name>Model other</name>
<retentionHours>0</retentionHours>
<dirPattern>(acars|airmet|binlighting|bufrmos|bufrua|bufsigwx|convsigmet|goessndg|manual|mdlsndg|poessndg|profiler|shef|text)/[^/]*/[^/]*/(.*)</dirPattern>
<display>\1 - \2</display>
<saveDir>(acars|airmet|binlighting|bufrmos|bufrua|bufsigwx|convsigmet|goessndg|manual|mdlsndg|poessndg|profiler|shef|text)/YYYYMMDD/HH/.*</saveDir>
<saveFiles>.*</saveFiles>
<dirPattern>(acars|airmet|binlightning|bufrmos|bufrua|bufrsigwx|convsigmet|goessndg|manual|mdlsndg|poessndg|profiler|shef|text)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<display>{1}</display>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</category>
<category>
<name>Satellite</name>
<retentionHours>101</retentionHours>
<dirPattern>sat/[^/]*/[^/]*/(.*)</dirPattern>
<display>\1</display>
<saveDir>sat/${YYYY}${MM}${DD}/${HH}/.*</saveDir>
<saveFiles>.*</saveFiles>
<dirPattern>sat/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<display>{5}</display>
<dateGroupIndices>1,2,3,4</dateGroupIndices>
</category>
</archive>