xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #2224 Changes to have DataSet in configuration." into development

Browse source 
Former-commit-id: d4a46b8283ac0cb47f62c43b999d7f119161e08d
This commit is contained in:
Richard Peter 2013-08-12 15:01:51 -05:00 • committed by Gerrit Code Review
commit d8124958b5
8 changed files with 694 additions and 437 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/data/SizeJob.java
View file

@ -40,6 +40,7 @@ import com.raytheon.uf.common.time.util.TimeUtil;
* ------------ ---------- ----------- --------------------------
* Jun 13, 2013 rferrel Initial creation
* Jul 24, 2013 #2220 rferrel Change to get all data sizes only one time.
* Aug 02, 2013 #2224 rferrel Changes for new configuration files.
* Aug 06, 2013 #2222 rferrel Changes to display all selected data.
*
* </pre>

219
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java
View file

@ -32,10 +32,11 @@ import java.util.Calendar;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.TreeSet;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -85,6 +86,7 @@ import com.raytheon.uf.common.util.FileUtil;
* Changed to use File.delete() instead of Apache FileUtil.deleteQuietly().
* Added warn logging for failure to delete.
* Jul 24, 2013 2221 rferrel Changes for select configuration.
* Aug 06, 2013 2224 rferrel Changes to use DataSet.
*
* </pre>
*
@ -529,91 +531,79 @@ public class ArchiveConfigManager {
*/
private List<File> getDisplayFiles(DisplayData displayData, long startTime,
long endTime) {
List<File> fileList = new LinkedList<File>();
ArchiveConfig archiveConfig = displayData.archiveConfig;
CategoryConfig categoryConfig = displayData.categoryConfig;
String[] indexValues = categoryConfig.getDateGroupIndices().split(
"\\s*,\\s*");
int yearIndex = Integer.parseInt(indexValues[0]);
int monthIndex = Integer.parseInt(indexValues[1]);
int dayIndex = Integer.parseInt(indexValues[2]);
int hourIndex = Integer.parseInt(indexValues[3]);
for (CategoryDataSet dataSet : displayData.dataSets) {
String filePatternStr = categoryConfig.getFilePattern();
int[] timeIndices = dataSet.getTimeIndices();
boolean dirOnly = (filePatternStr == null)
|| ".*".equals(filePatternStr);
String filePatternStr = dataSet.getFilePattern();
List<File> dirs = displayData.dirs;
boolean dirOnly = dataSet.isDirOnly();
int beginIndex = archiveConfig.getRootDir().length();
List<File> dirs = displayData.dirsMap.get(dataSet);
Calendar fileCal = TimeUtil.newCalendar();
fileCal.setTimeZone(TimeZone.getTimeZone("UTC"));
int beginIndex = archiveConfig.getRootDir().length();
List<File> fileList = new ArrayList<File>();
Calendar fileCal = TimeUtil.newCalendar();
fileCal.setTimeZone(TimeZone.getTimeZone("UTC"));
if (dirOnly) {
for (String dirPattern : categoryConfig.getDirPatternList()) {
Pattern pattern = Pattern.compile(dirPattern);
if (dirOnly) {
for (String dirPattern : dataSet.getDirPatterns()) {
Pattern pattern = dataSet.getPattern(dirPattern);
for (File dir : dirs) {
String path = dir.getAbsolutePath().substring(beginIndex);
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
int year = Integer.parseInt(matcher.group(yearIndex));
// Adjust month value to Calendar's 0 - 11
int month = Integer.parseInt(matcher.group(monthIndex)) - 1;
int day = Integer.parseInt(matcher.group(dayIndex));
int hour = Integer.parseInt(matcher.group(hourIndex));
fileCal.set(year, month, day, hour, 0, 0);
long fileTime = fileCal.getTimeInMillis();
if ((startTime <= fileTime) && (fileTime < endTime)) {
fileList.add(dir);
}
}
}
}
} else {
for (String dirPattern : categoryConfig.getDirPatternList()) {
Pattern pattern = Pattern.compile(dirPattern + File.separator
+ categoryConfig.getFilePattern());
final Pattern filePattern = Pattern.compile("^"
+ filePatternStr + "$");
for (File dir : dirs) {
List<File> fList = FileUtil.listDirFiles(dir,
new FileFilter() {
@Override
public boolean accept(File pathname) {
return filePattern.matcher(
pathname.getName()).matches();
}
}, false);
for (File file : fList) {
String path = file.getAbsolutePath().substring(
for (File dir : dirs) {
String path = dir.getAbsolutePath().substring(
beginIndex);
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
int year = Integer.parseInt(matcher
.group(yearIndex));
// Adjust month value to Calendar's 0 - 11
int month = Integer.parseInt(matcher
.group(monthIndex)) - 1;
int day = Integer.parseInt(matcher.group(dayIndex));
int hour = Integer.parseInt(matcher
.group(hourIndex));
fileCal.set(year, month, day, hour, 0, 0);
long fileTime = fileCal.getTimeInMillis();
Long fileTime = dataSet.getMatchTimeInMilliseconds(
timeIndices, matcher);
if (fileTime == null) {
fileTime = dir.lastModified();
}
if ((startTime <= fileTime) && (fileTime < endTime)) {
fileList.add(file);
fileList.add(dir);
}
}
}
}
} else {
for (String dirPattern : dataSet.getDirPatterns()) {
Pattern pattern = dataSet.getPattern(dirPattern);
final Pattern filePattern = Pattern.compile("^"
+ filePatternStr + "$");
for (File dir : dirs) {
List<File> fList = FileUtil.listDirFiles(dir,
new FileFilter() {
@Override
public boolean accept(File pathname) {
return filePattern.matcher(
pathname.getName()).matches();
}
}, false);
for (File file : fList) {
String path = file.getAbsolutePath().substring(
beginIndex);
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
Long timestamp = dataSet
.getMatchTimeInMilliseconds(
timeIndices, matcher);
long fileTime = timestamp == null ? file
.lastModified() : timestamp.longValue();
if ((startTime <= fileTime)
&& (fileTime < endTime)) {
fileList.add(file);
}
}
}
}
}
}
}
return fileList;
}
@ -625,15 +615,13 @@ public class ArchiveConfigManager {
* @param categoryConfig
* @return dirs
*/
private List<File> getDirs(ArchiveConfig archiveConfig,
CategoryConfig categoryConfig) {
private List<File> getDirs(File rootFile, CategoryDataSet dataSet) {
List<File> resultDirs = new ArrayList<File>();
File rootFile = new File(archiveConfig.getRootDir());
List<File> dirs = new ArrayList<File>();
List<File> tmpDirs = new ArrayList<File>();
List<File> swpDirs = null;
for (String dirPattern : categoryConfig.getDirPatternList()) {
for (String dirPattern : dataSet.getDirPatterns()) {
String[] subExpr = dirPattern.split(File.separator);
dirs.clear();
dirs.add(rootFile);
@ -679,50 +667,59 @@ public class ArchiveConfigManager {
Map<String, List<File>> displayMap = new HashMap<String, List<File>>();
ArchiveConfig archiveConfig = archiveMap.get(archiveName);
String rootDirName = archiveConfig.getRootDir();
CategoryConfig categoryConfig = findCategory(archiveConfig,
categoryName);
List<String> dirPatternList = categoryConfig.getDirPatternList();
File rootFile = new File(rootDirName);
TreeMap<String, DisplayData> displays = new TreeMap<String, DisplayData>();
for (CategoryDataSet dataSet : categoryConfig.getDataSetList()) {
List<String> dataSetDirPatterns = dataSet.getDirPatterns();
// index for making directory paths' relative to the root path.
List<File> dirs = getDirs(archiveConfig, categoryConfig);
List<File> dirs = getDirs(rootFile, dataSet);
File rootFile = new File(archiveMap.get(archiveName).getRootDir());
int beginIndex = rootFile.getAbsolutePath().length() + 1;
List<Pattern> patterns = new ArrayList<Pattern>(dirPatternList.size());
int beginIndex = rootFile.getAbsolutePath().length() + 1;
List<Pattern> patterns = new ArrayList<Pattern>(
dataSetDirPatterns.size());
for (String dirPattern : dirPatternList) {
Pattern pattern = Pattern.compile("^" + dirPattern + "$");
patterns.add(pattern);
}
for (String dirPattern : dataSetDirPatterns) {
Pattern pattern = Pattern.compile("^" + dirPattern + "$");
patterns.add(pattern);
}
TreeSet<String> displays = new TreeSet<String>(
String.CASE_INSENSITIVE_ORDER);
MessageFormat msgfmt = new MessageFormat(dataSet.getDisplayLabel());
StringBuffer sb = new StringBuffer();
FieldPosition pos0 = new FieldPosition(0);
MessageFormat msgfmt = new MessageFormat(categoryConfig.getDisplay());
StringBuffer sb = new StringBuffer();
FieldPosition pos0 = new FieldPosition(0);
for (File dir : dirs) {
String path = dir.getAbsolutePath().substring(beginIndex);
for (Pattern pattern : patterns) {
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
sb.setLength(0);
String[] args = new String[matcher.groupCount() + 1];
args[0] = matcher.group();
for (int i = 1; i < args.length; ++i) {
args[i] = matcher.group(i);
for (File dir : dirs) {
String path = dir.getAbsolutePath().substring(beginIndex);
for (Pattern pattern : patterns) {
Matcher matcher = pattern.matcher(path);
if (matcher.matches()) {
sb.setLength(0);
String[] args = new String[matcher.groupCount() + 1];
args[0] = matcher.group();
for (int i = 1; i < args.length; ++i) {
args[i] = matcher.group(i);
}
String displayLabel = msgfmt.format(args, sb, pos0)
.toString();
List<File> displayDirs = displayMap.get(displayLabel);
if (displayDirs == null) {
displayDirs = new ArrayList<File>();
displayMap.put(displayLabel, displayDirs);
}
displayDirs.add(dir);
DisplayData displayData = displays.get(displayLabel);
if (displayData == null) {
displayData = new DisplayData(archiveConfig,
categoryConfig, dataSet, displayLabel);
displays.put(displayLabel, displayData);
} else {
displayData.dataSets.add(dataSet);
}
displayData.dirsMap.put(dataSet, displayDirs);
break;
}
String displayLabel = msgfmt.format(args, sb, pos0)
.toString();
List<File> displayDirs = displayMap.get(displayLabel);
if (displayDirs == null) {
displayDirs = new ArrayList<File>();
displayMap.put(displayLabel, displayDirs);
}
displayDirs.add(dir);
displays.add(displayLabel);
break;
}
}
}
@ -730,15 +727,7 @@ public class ArchiveConfigManager {
List<DisplayData> displayDataList = new ArrayList<DisplayData>(
displays.size());
for (String displayLabel : displays) {
DisplayData displayData = new DisplayData(archiveConfig,
categoryConfig, displayLabel, displayMap.get(displayLabel));
if (setSelect) {
displayData.setSelected(categoryConfig
.getSelectedDisplayNames().contains(displayLabel));
}
displayDataList.add(displayData);
}
displayDataList.addAll(displays.values());
return displayDataList;
}

165
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java
View file

@ -39,10 +39,13 @@ import javax.xml.bind.annotation.XmlRootElement;
* &lt;name>redbook&lt;/name>
* &lt;!-- When 0 default to the parent archive's retentionHours -->
* &lt;retentionHours>0&lt;/retentionHours>
* &lt;dirPattern>hdf5/(redbook)&lt;/dirPattern>
* &lt;displayLabel>{1}&lt;/displayLabel>
* &lt;filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*&lt;/filePattern>
* &lt;dateGroupIndices>2,3,4,5&lt;/dateGroupIndices>
* &lt;dataSet>
* &lt;dirPattern>hdf5/(redbook)&lt;/dirPattern>
* &lt;displayLabel>{1}&lt;/displayLabel>
* &lt;filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*&lt;/filePattern>
* &lt;timeType>Date&lt;/timeType>
* &lt;dateGroupIndices>2,3,4,5&lt;/dateGroupIndices>
* &lt;/dataSet>
* &lt;/category>
* </pre>
*
@ -52,9 +55,12 @@ import javax.xml.bind.annotation.XmlRootElement;
* &lt;category>
* &lt;name>Model grib&lt;/name>
* &lt;retentionHours>0&lt;/retentionHours>
* &lt;dirPattern>grib/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)&lt;/dirPattern>
* &lt;displayLabel>{5}&lt;/displayLabel>
* &lt;dateGroupIndices>1,2,3,4&lt;/dateGroupIndices>
* &lt;dataSet>
* &lt;dirPattern>grib/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)&lt;/dirPattern>
* &lt;displayLabel>{5}&lt;/displayLabel>
* &lt;timeType>Date&lt;/timeType>
* &lt;dateGroupIndices>1,2,3,4&lt;/dateGroupIndices>
* &lt/dataSet>
* &lt;/category>
* </pre>
*
@ -65,6 +71,7 @@ import javax.xml.bind.annotation.XmlRootElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* May 1, 2013 1966 rferrel Initial creation
* Aug 03, 2013 2224 rferrel Changes to include DataSet.
*
* </pre>
*
@ -88,63 +95,8 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
@XmlElement(name = "extRetentionHours")
private int retentionHours;
/**
* A regex pattern to find directories controlled by the category. These
* directories should be relative to the parent archive's directory. For
* example:
*
* <pre>
* &lt;dirPattern>grib2/\d{8}/\d{2}/(.*)/&lt;/dirPattern>
* </pre>
*/
@XmlElement(name = "dirPattern")
private List<String> dirPatternList;
/**
* Use to display the information found by the dirPattern. Any groups in the
* dirPatern may be displayed. For example:
*
* <pre>
* &lt;dirName>(grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)&lt;/dirName>
* &lt;displayLabel>{1} - {6}&lt;/displayLabel>
* </pre>
*
* The {1} will be replaced by the first group (grib2) in the regex
* expression in dirName. The {6} is the sixth group (.*). {0} is the whole
* expression match.
*/
@XmlElement(name = "displayLabel")
private String display;
/**
* A comma separated list of 4 numbers representing the group indices
* specifying the location of the numeric date time stamp. The indices must
* be in the order of year, month, day and hour. The group numbering starts
* with the first group in the dirPattern and continues with any grouping in
* the filePattern.
*
* <pre>
* &lt;dirPattern>hdf5/(redbook)&lt;/dirPattern>
* &lt;displayLabel>{1}&lt;/displayLabel>
* &lt;filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*&lt;/filePattern>
* &lt;dateGroupIndices>2,3,4,5&lt;/dateGroupIndices>
* </pre>
*/
@XmlElement(name = "dateGroupIndices")
private String dateGroupIndices;
/**
* A saveDir directory may contain files with data for several days. This
* allows any of the year, month, day and hour to be part of a file name.
* Default is all files in the directory. For example:
*
* <pre>
* &lt;saveDir>hd5/redbook/(^/]*&#47/)&lt/saveDir>
* &lt;saveFile>redbook-${YYYY}-${MM}-${DD}-${HH}\..*&lt;saveFiles>
* </pre>
*/
@XmlElement(name = "filePattern")
private String filePattern;
@XmlElement(name = "dataSet")
private List<CategoryDataSet> dataSetList;
@XmlElement(name = "selectedDisplayName")
private final Collection<String> selectedDisplayNames = new TreeSet<String>();
@ -188,76 +140,12 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
this.retentionHours = retentionHours;
}
/**
* Obtain the list of directory patterns.
*
* @return dirPatternList
*/
public List<String> getDirPatternList() {
return new ArrayList<String>(dirPatternList);
public List<CategoryDataSet> getDataSetList() {
return new ArrayList<CategoryDataSet>(dataSetList);
}
/**
* Set the directory pattern list; must not be null.
*
* @param dirPatternList
*/
public void setDirPatternList(List<String> dirPatternList) {
this.dirPatternList = dirPatternList;
}
/**
* Get the display label pattern.
*
* @return display
*/
public String getDisplay() {
return display == null ? "" : display;
}
/**
* Set the display label pattern.
*
* @param display
*/
public void setDisplay(String display) {
this.display = display;
}
/**
* Get the save directory pattern..
*
* @return dateGroups
*/
public String getDateGroupIndices() {
return dateGroupIndices;
}
/**
* Set the save directory pattern; must not be null.
*
* @param saveDir
*/
public void setDateGroupIndices(String dateGroupIndices) {
this.dateGroupIndices = dateGroupIndices;
}
/**
* Get the save files pattern.
*
* @return saveFiles
*/
public String getFilePattern() {
return filePattern;
}
/**
* Set the save files pattern; may be null.
*
* @param saveFiles
*/
public void setFilePattern(String filePattern) {
this.filePattern = filePattern;
public void setDataSetList(List<CategoryDataSet> dataSetList) {
this.dataSetList = dataSetList;
}
public Collection<String> getSelectedDisplayNames() {
@ -285,9 +173,9 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
*/
@Override
public int compareTo(CategoryConfig o) {
return getDisplay().compareToIgnoreCase(o.getDisplay());
return getName().compareToIgnoreCase(o.getName());
}
/*
* (non-Javadoc)
*
@ -298,13 +186,10 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
StringBuilder sb = new StringBuilder();
sb.append("Category [ name: ").append(getName());
sb.append(", retentionHours: ").append(getRetentionHours());
sb.append(", dirPatternList[ ");
for (String dirPattern : getDirPatternList()) {
sb.append(" \"").append(dirPattern).append("\",");
sb.append(", dataSetList[ ");
for (CategoryDataSet dataSet : getDataSetList()) {
sb.append(dataSet).append(", ");
}
sb.append("], filePattern: ").append(getFilePattern());
sb.append(", displayLabel: ").append(getDisplay());
sb.append(", dateGroupIndices: ").append(getDateGroupIndices());
sb.append(", selectedDisplayNames: ");
if (selectedDisplayNames == null) {
sb.append("null");

280
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryDataSet.java Normal file
View file

@ -0,0 +1,280 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.archive.config;
import java.io.File;
import java.util.Calendar;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.time.util.TimeUtil;
/**
* A grouping of data set information used in a category.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Aug 6, 2013 #2224 rferrel Initial creation
*
* </pre>
*
* @author rferrel
* @version 1.0
*/
@XmlAccessorType(XmlAccessType.NONE)
@XmlRootElement(name = "dataSet")
public class CategoryDataSet {
public static final int YEAR_INDEX = 0;
public static final int MONTH_INDEX = 1;
public static final int DAY_INDEX = 2;
public static final int HOUR_INDEX = 3;
public static final int TIMESTAMP_INDEX = 0;
/**
* Types of times and the number of indices for getting the time stamp from
* patterns.
*/
public static enum TimeType {
Date(4), EpochSec(1), EpochMS(1), File(0);
private final int numIndices;
private TimeType(int numIndices) {
this.numIndices = numIndices;
}
public int getNumGroupIndices() {
return numIndices;
}
}
/**
* List of directory patterns.
*/
@XmlElement(name = "dirPattern")
private List<String> dirPatterns;
/**
* Optional file pattern.
*/
@XmlElement(name = "filePattern")
private String filePattern;
@XmlElement(name = "timeType")
private TimeType timeType;
/**
* The display label.
*/
@XmlElement(name = "displayLabel")
private String displayLabel;
private int[] timeIndices = null;
/**
* The index with group number for getting desired parts of the TimeType.
*/
@XmlElement(name = "dateGroupIndices")
private String dateGroupIndices = "";
public List<String> getDirPatterns() {
return dirPatterns;
}
public void setDirPatterns(List<String> dirPatterns) {
this.dirPatterns = dirPatterns;
}
public String getFilePattern() {
return filePattern;
}
public void setFilePattern(String filePattern) {
this.filePattern = filePattern;
}
public TimeType getTimeType() {
return timeType;
}
public void setTimeType(TimeType timeType) {
this.timeType = timeType;
}
public String getDisplayLabel() {
return displayLabel;
}
public void setDisplayLabel(String displayLabel) {
this.displayLabel = displayLabel;
}
public String getDateGroupIndices() {
return dateGroupIndices;
}
public void setDateGroupIndices(String dateGroupIndices) {
this.dateGroupIndices = dateGroupIndices;
this.timeIndices = null;
}
/**
* Get the array of time indices based on time type and date group indices.
*
* @return timeIndices
*/
public int[] getTimeIndices() {
if (timeIndices == null) {
timeIndices = new int[timeType.getNumGroupIndices()];
if (timeIndices.length > 0) {
String[] indexValues = getDateGroupIndices().split("\\s*,\\s*");
for (int index = 0; index < timeIndices.length; ++index) {
timeIndices[index] = Integer.parseInt(indexValues[index]);
}
}
}
return timeIndices;
}
/**
* Get the Pattern for dirPattern.
*
* @param dirPattern
*
* @return pattern or null if dirPattern not in the list of directory
* patterns.
*/
public Pattern getPattern(String dirPattern) {
Pattern pattern = null;
if (dirPatterns.contains(dirPattern)) {
if (isDirOnly()) {
pattern = Pattern.compile(dirPattern);
} else {
pattern = Pattern.compile(dirPattern + File.separator
+ getFilePattern());
}
}
return pattern;
}
/**
*
* @return true when only the dirPatterns should be used.
*/
public boolean isDirOnly() {
return filePattern == null || filePattern.equals(".*");
}
/**
* Get time stamp for file based on timetype. Assumes file path matches
*
* @param timeIndices
* @param matcher
* @param file
* @return
*/
public Long getMatchTimeInMilliseconds(int[] timeIndices, Matcher matcher) {
return CategoryDataSet.getMatchTimeInMilliseconds(timeType,
timeIndices, matcher);
}
/**
* Get file time based on time type.
*
* @param timeType
* @param timeIndices
* @param matcher
* @param file
* @return fileTime
*/
public static Long getMatchTimeInMilliseconds(
CategoryDataSet.TimeType timeType, int[] timeIndices,
Matcher matcher) {
Long fileTime = null;
switch (timeType) {
case Date:
Calendar fileCal = TimeUtil.newGmtCalendar();
int year = Integer.parseInt(matcher
.group(timeIndices[CategoryDataSet.YEAR_INDEX]));
// Adjust month value to Calendar's 0 - 11
int month = Integer.parseInt(matcher
.group(timeIndices[CategoryDataSet.MONTH_INDEX])) - 1;
int day = Integer.parseInt(matcher
.group(timeIndices[CategoryDataSet.DAY_INDEX]));
int hour = Integer.parseInt(matcher
.group(timeIndices[CategoryDataSet.HOUR_INDEX]));
fileCal.set(year, month, day, hour, 0, 0);
fileTime = fileCal.getTimeInMillis();
break;
case EpochMS:
fileTime = Long.parseLong(matcher
.group(timeIndices[CategoryDataSet.TIMESTAMP_INDEX]));
break;
case EpochSec:
fileTime = Long.parseLong(matcher
.group(timeIndices[CategoryDataSet.TIMESTAMP_INDEX]));
fileTime *= TimeUtil.MILLIS_PER_SECOND;
break;
case File:
fileTime = null;
break;
default:
fileTime = null;
break;
}
return fileTime;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
StringBuilder sb = new StringBuilder("DataSet[ ");
sb.append("TimeType: ").append(getTimeType());
sb.append("dateGroupIndices: ").append(getDateGroupIndices());
sb.append(", isDirOnly: ").append(isDirOnly());
sb.append(", displayLabel: ").append(getDisplayLabel());
sb.append(", dirPatterns[ ");
for (String dirPattern : getDirPatterns()) {
sb.append(dirPattern).append(", ");
}
sb.append("], filePattern: ").append(
filePattern == null ? "null" : filePattern);
sb.append("]");
return sb.toString();
}
}

119
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java
View file

@ -40,6 +40,7 @@ import com.raytheon.uf.common.time.util.TimeUtil;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 21, 2013 1965 bgonzale Initial creation
* Aug 03, 2013 2224 rferrel Changes for new configuration files.
*
* </pre>
*
@ -56,13 +57,11 @@ public class CategoryFileDateHelper implements IFileDateHelper {
private final Pattern categoryTopLevelDirPattern;
private final int yearIndex;
private final CategoryDataSet.TimeType timeType;
private final int monthIndex;
private final boolean isDirOnly;
private final int dayIndex;
private final int hourIndex;
private final int[] timeIndices;
/**
* Initialization constructor.
@ -76,13 +75,13 @@ public class CategoryFileDateHelper implements IFileDateHelper {
*/
public CategoryDateInfo(Pattern datePattern,
Pattern categoryTopLevelDirPattern,
int yearIndex, int monthIndex, int dayIndex, int hourIndex) {
CategoryDataSet.TimeType timeType, boolean isDirOnly,
int[] timeIndices) {
this.datePattern = datePattern;
this.categoryTopLevelDirPattern = categoryTopLevelDirPattern;
this.yearIndex = yearIndex;
this.monthIndex = monthIndex;
this.dayIndex = dayIndex;
this.hourIndex = hourIndex;
this.timeType = timeType;
this.isDirOnly = isDirOnly;
this.timeIndices = timeIndices;
}
}
@ -91,8 +90,6 @@ public class CategoryFileDateHelper implements IFileDateHelper {
private final String rootDir;
private final boolean isDirOnly;
/**
* Initialization constructor.
*
@ -101,38 +98,37 @@ public class CategoryFileDateHelper implements IFileDateHelper {
* categoryTopLevelDirPattern
*/
public CategoryFileDateHelper(CategoryConfig config, String rootDir) {
List<String> categoryDirPatternList = config.getDirPatternList();
this.dateInfoList = new ArrayList<CategoryFileDateHelper.CategoryDateInfo>(
categoryDirPatternList.size());
String filePatternStr = config.getFilePattern();
this.rootDir = rootDir;
this.isDirOnly = (filePatternStr == null)
|| ".*".equals(filePatternStr);
List<CategoryDataSet> categoryDataSetList = config.getDataSetList();
int size = 0;
for (CategoryDataSet dataSet : categoryDataSetList) {
size += dataSet.getDirPatterns().size();
}
for (String patternString : categoryDirPatternList) {
Pattern datePattern = null;
if (isDirOnly) {
datePattern = Pattern.compile(patternString);
} else {
datePattern = Pattern.compile(patternString + File.separator
+ config.getFilePattern());
this.dateInfoList = new ArrayList<CategoryFileDateHelper.CategoryDateInfo>(
size);
boolean isDirOnly;
CategoryDataSet.TimeType timeType;
for (CategoryDataSet dataSet : categoryDataSetList) {
isDirOnly = dataSet.isDirOnly();
timeType = dataSet.getTimeType();
for (String patternString : dataSet.getDirPatterns()) {
Pattern datePattern = dataSet.getPattern(patternString);
int dirSeparatorIndex = patternString
.indexOf(File.separatorChar);
patternString = dirSeparatorIndex > patternString.length()
|| dirSeparatorIndex < 0 ? patternString
: patternString.substring(0, dirSeparatorIndex);
Pattern categoryTopLevelDirPattern = Pattern
.compile(patternString);
int[] timeIndices = dataSet.getTimeIndices();
dateInfoList.add(new CategoryDateInfo(datePattern,
categoryTopLevelDirPattern, timeType, isDirOnly,
timeIndices));
}
int dirSeparatorIndex = patternString.indexOf(File.separatorChar);
patternString = dirSeparatorIndex > patternString.length()
|| dirSeparatorIndex < 0 ? patternString : patternString
.substring(0, dirSeparatorIndex);
Pattern categoryTopLevelDirPattern = Pattern.compile(patternString);
String[] indexValues = config.getDateGroupIndices().split(
"\\s*,\\s*");
int yearIndex = Integer.parseInt(indexValues[0]);
int monthIndex = Integer.parseInt(indexValues[1]);
int dayIndex = Integer.parseInt(indexValues[2]);
int hourIndex = Integer.parseInt(indexValues[3]);
dateInfoList.add(new CategoryDateInfo(datePattern,
categoryTopLevelDirPattern, yearIndex, monthIndex,
dayIndex, hourIndex));
}
}
@ -145,36 +141,39 @@ public class CategoryFileDateHelper implements IFileDateHelper {
*/
@Override
public Calendar getFileDate(String filenamePath) {
String pathForPatternCheck = filenamePath.substring(rootDir.length());
pathForPatternCheck = isDirOnly ? FilenameUtils
.getFullPathNoEndSeparator(pathForPatternCheck)
: pathForPatternCheck;
String pathForFilePatternCheck = filenamePath.substring(rootDir
.length());
String pathForDirPatternCheck = FilenameUtils
.getFullPathNoEndSeparator(pathForFilePatternCheck);
Calendar result = null;
Long timestamp = null;
for (CategoryDateInfo dateInfo : dateInfoList) {
Matcher matcher = dateInfo.datePattern.matcher(pathForPatternCheck);
Matcher matcher = null;
if (dateInfo.isDirOnly) {
matcher = dateInfo.datePattern.matcher(pathForDirPatternCheck);
} else {
matcher = dateInfo.datePattern.matcher(pathForFilePatternCheck);
}
if (matcher.matches()) {
int year = Integer.parseInt(matcher.group(dateInfo.yearIndex));
// Adjust month value to Calendar's 0 - 11
int month = Integer
.parseInt(matcher.group(dateInfo.monthIndex)) - 1;
int day = Integer.parseInt(matcher.group(dateInfo.dayIndex));
int hour = Integer.parseInt(matcher.group(dateInfo.hourIndex));
result = TimeUtil.newGmtCalendar();
result.set(year, month, day, hour, 0, 0);
timestamp = CategoryDataSet.getMatchTimeInMilliseconds(
dateInfo.timeType, dateInfo.timeIndices, matcher);
break;
}
}
if (result == null) {
if (timestamp == null) {
// no matching pattern, use file last modified date
File file = new File(filenamePath);
long lastModifiedMillis = file.lastModified();
result = TimeUtil.newGmtCalendar();
result.setTimeInMillis(lastModifiedMillis);
timestamp = file.lastModified();
}
// TODO future speed improvement refactor IFileDateHelper to have a
// method that returns a long instead of Calendar. That will prevent
// converting Calendar to long then back to a Calendar.
result = TimeUtil.newGmtCalendar();
result.setTimeInMillis(timestamp);
return result;
}

25
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/DisplayData.java
View file

@ -1,8 +1,11 @@
package com.raytheon.uf.common.archive.config;
import java.io.File;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.raytheon.uf.common.util.SizeUtil;
@ -18,6 +21,7 @@ import com.raytheon.uf.common.util.SizeUtil;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 7, 2013 1966 rferrel Initial creation
* Aug 02, 2013 2224 rferrel Changes to include DataSet in configuration.
* Aug 06, 2013 2222 rferrel Changes to display all selected data.
*
* </pre>
@ -71,22 +75,26 @@ public class DisplayData implements Comparable<DisplayData> {
/** The data's category configuration. */
protected final CategoryConfig categoryConfig;
protected final List<CategoryDataSet> dataSets = new ArrayList<CategoryDataSet>();
/** The display label for this data. */
protected final String displayLabel;
/**
* List of directories for the display label matching the category's
* directory pattern and found under the archive's root directory.
* Mappings of a list of directories for the display label matching the data
* set's directory patterns and found under the archive's root directory.
*/
protected final List<File> dirs;
protected final Map<CategoryDataSet, List<File>> dirsMap = new HashMap<CategoryDataSet, List<File>>();
/**
* For use by GUI to indicate. Use to indicate selected for retention or for
* placing in a case.
* For use by GUI to indicate display label's row is selected.
*/
private boolean selected = false;
/** For use by GUI for indicating the size of the directories' contents. */
/**
* For use by GUI for indicating the size of the display label's row
* contents.
*/
private long size = UNKNOWN_SIZE;
/**
@ -94,15 +102,14 @@ public class DisplayData implements Comparable<DisplayData> {
*
* @param archiveConfig
* @param categoryConfig
* @param dataSet
* @param displayLabel
* @param dirs
*/
public DisplayData(ArchiveConfig archiveConfig,
CategoryConfig categoryConfig, String displayLabel, List<File> dirs) {
CategoryConfig categoryConfig, CategoryDataSet dataSet, String displayLabel) {
this.archiveConfig = archiveConfig;
this.categoryConfig = categoryConfig;
this.displayLabel = displayLabel;
this.dirs = dirs;
}
/**

182
edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archiver/purger/PROCESSED_DATA.xml
View file

@ -24,43 +24,60 @@
* Date Ticket# Engineer Description
* ============ ========== =========== ==========================
* Jun 20, 2013 1966 rferrel Initial creation
* Aug 05, 2012 2224 rferrel Changes to add dataSet tags.
*
* @author rferrel
* @version 1.0
-->
<!--
The <archive> contains five types of tags:
The <archive> contains four types of tags:
<name> - Required tag. The id for the archive such as Raw or Processed.
Used in the GUIs to filter which archive to display.
<rootDir> - Required tag. The root directory on the edex server for the archive.
<minRetentionHours> - Required tag. The default number of hours to retain data in the <rootDir>
<category> - Logical grouping of archive's sub-directories.
The <category> has six types of tags:
The <category> has four types of tags:
<name> - Required tag. The id for the category such as grib2 or redbook.
Used in the GUIs to filter what is the display in the table.
<extRetentionHours> - Optional tag. The extended retentionHours for selected directories of this category.
Default is the archive's <minRetentionHours>.
<dirPattern> - Required tag. A regex pattern for finding directories for this category.
<dataSet> - Required to have a least one. Each one contains a set of tags explained below.
<selectedDisplayNames> - A directory matching <dirPattern>. These are selected directories from the Retention GUI. The purger
will used the category's <extRetentionHours> instead of the Arhivie's <minRetentionHours>.
An optional tag; may have more then one. (NOTE these are set internally when a selection configuration
file is loaded. They should not appear in the configuration file.)
The <dataSet> contains ? types of tags:
<dirPattern> - Required to have at least one. A regex pattern for finding directories for this category.
The pattern is relative to the archive's <rootDir>. Wildcard patterns do not cross directory
delimiter /. Thus to match 3 levels of directories you would need .*/.*/.* .
See patterns and groups section. There may be more then one of these tags. The restriction is they
must all have the same number of groupings and be in the same order.
See patterns and groups section. There may be more then one of these tags in a <dataSet>. The
restriction is they must all have the same number of groupings and be in the same order to match up
with the <displayLabel>'s, and <dateGroupIndicies>'s values.
<filePattern> - Optional tag. A pattern to find files in the directories that match the <dirPattern>.
Default is everything in the directories that match <dirPattern>.
See patterns and groups section.
<displayLabel> - Required tag. The label to display for directories that match <dirPattern>. Any group in the
<dirPattern> may be made part of the label by placing the group's index inside parenthesis, {1}.
More then one directory may match the <dirPattern>. The archive GUIs may collapse them into a
More then one directory may match the <dirPattern>. The archive GUIs may collapse them into a
single table entry.
<dateGroupIndicies> - Required tag. A comma separated list of 4 numbers which are the index of the groups in
<dirPattern> and/or <filePattern> that contain in order the year, mouth, day and hour information.
<timeType> - Optional tag to determine what type of time stamp is being used to get files/directories for retention
and case creation. The value dictates how many groupings in the <dirPattern>s and/or <filePattern> are
used to get the time stamp for a file. The four values are:
Date - (default) the time stamp is made up of four groups in the patterns: year, month, day and hour.
EpochSec - The time stamp has one group in the patterns which is the epoch time in seconds.
EpochMS - The time stamp has one group in the patterns which is the epoch time in milliseconds.
File - No group is used to get the time stamp. Instead use the files date of last modification.
<dateGroupIndicies> - Required tag when <timeType> has any value but File.
Date - A comma separated list of 4 numbers which are in order the index for year, month, day and hour.
EpochSec - A number which is the index for the epoch in seconds.
EpochMS - A number which is the index for the epoch in milliseconds.
File - Not needed since no group is used to get the time stamp.
This is used to determine what files/directories to retain or a range of directories/files to copy
for case creation. Note to get the group's index the <dirPattern> and <filePattern> are combined.
Thus if there are 5 groups in the <dirPattern> then the first group in the <filePattern> is index 6.
<selectedDisplayNames> - A directory matching <dirPattern>. These are selected directories from the Retention GUI. The purger
will used the category's <extRetentionHours> instead of the Arhivie's <minRetentionHours>.
An optional tag; may have more then one.
Patterns and groups.
The <dirPattern> and <filePattern> use Java regex expressions; similar to the ldm's pqact.conf file.
For more details see http://docs.oracle.com/javase/tutorial/essential/regex/
@ -77,7 +94,7 @@
Finally the sixth group will match any sub-directory that in the hour directory.
Thus the directory paths <rootPath>/grib2/20130527/18/GFS will generate the display string, grib2 - GFS, and from the grouping we
can find the year, 201; month, 05; day, 27 and hour, 18.
can find the year, 2013; month, 05; day, 27 and hour, 18.
Example with <filePattern>:
<dirPattern>hdf5/(redbook)</dirPattern>
@ -89,11 +106,14 @@
<category>
<name>Observation</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(acars|airep|airmet|taf)</dirPattern>
<dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<dataSet>
<dirPattern>(acars|airep|airmet|taf)</dirPattern>
<dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
<displayLabel>{1}</displayLabel>
<timeType>Date</timeType>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</dataSet>
</category>
The first <dirPattern> looks for files matching the <filePattern> in the directories acars, airep, airmet or taf.
The second <dirPattern> expects to find the files in subdirectories of bufrsigwx or sfcobs such as bufrsigwx/SWH.
@ -110,79 +130,111 @@
<rootDir>/awips2/edex/data/archive/</rootDir>
<minRetentionHours>24</minRetentionHours>
<category>
<name>DAT</name>
<name>Decision Assistance</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(cwat|fog|ffmp|preciprate|qpf|scan|vil)</dirPattern>
<filePattern>.*(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<dataSet>
<dirPattern>(cwat|fog|ffmp|fssobs|preciprate|qpf|scan|vil)</dirPattern>
<filePattern>.*(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</dataSet>
</category>
<category>
<name>gfe</name>
<name>GFE</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(gfe)/(.*)/.*</dirPattern>
<dirPattern>(gfe)/(.*)</dirPattern>
<filePattern>.*_(\d{4})(\d{2})(\d{2})_(\d{2}).*</filePattern>
<displayLabel>{2}</displayLabel>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
<dataSet>
<dirPattern>(gfe)/(.*)/.*</dirPattern>
<dirPattern>(gfe)/(.*)</dirPattern>
<filePattern>.*_(\d{4})(\d{2})(\d{2})_(\d{2}).*</filePattern>
<displayLabel>{2}</displayLabel>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
</dataSet>
</category>
<category>
<name>Local</name>
<extRetentionHours>168</extRetentionHours>
<dataset>
<dirPattern>(ldadhydro|ldadmesonet|ldadprofiler|ldad_manual|mesowest|qc)</dirPattern>
<filePattern>.*(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</dataset>
</category>
<category>
<name>Model</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(grid)/(.*)/(.*)</dirPattern>
<displayLabel>{2}</displayLabel>
<dateGroupIndices>4,5,6,7</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})-.*</filePattern>
<dataSet>
<dirPattern>(grid)/(.*)/(.*)</dirPattern>
<displayLabel>{2}</displayLabel>
<dateGroupIndices>4,5,6,7</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})-.*</filePattern>
</dataSet>
<dataSet>
<dirPattern>(modelsounding)/(.*)</dirPattern>
<dirPattern>(bufrmos)(.*)</dirPattern>
<displayLabel>{1} - {2}</displayLabel>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
<filePattern>.*(\d{4})-(\d{2})-(\d{2})[-_](\d{2}).*</filePattern>
</dataSet>
</category>
<category>
<name>Misc</name>
<name>Products</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(binlightning|ccfp|cwa|idft|lsr|manual|svrwx|tcg|tcm|tcs|text|warning|wcp)</dirPattern>
<dirPattern>(redbook)/.*</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<dataSet>
<dirPattern>(airmet|atcf|aww|bufrncwf|ccfp|convsigmet|cwa|ffg|intlsigmet|nonconvsigmet|stormtrack|taf|tcg|tcm|tcs|text|vaa|warning|wcp)</dirPattern>
<dirPattern>(bufrsigwx|redbook)/.*</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</dataSet>
</category>
<category>
<name>Observation</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(acars|airep|airmet|bufrascat|bufrhdw|bufrmthdw|bufrncwf|bufrssmi|convsigmet|fssobs|intlsigmet|nonconvsigmet|obs|pirep|taf)</dirPattern>
<dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<dataSet>
<dirPattern>(acars|airep|binlightning|bufrascat|bufrhdw|bufrmthdw|bufrssmi|idft|lsr|obs|pirep|recco|svrwx)</dirPattern>
<dirPattern>(sfcobs)/.*</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</dataSet>
</category>
<category>
<name>Satellite</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>satellite/(.*)/(.*)</dirPattern>
<filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
<displayLabel>{1}</displayLabel>
<dataSet>
<dirPattern>satellite/(.*)/(.*)</dirPattern>
<filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
<displayLabel>{1}</displayLabel>
</dataSet>
<dataSet>
<!-- Guess for mcidas and viirs is based on old example. -->
<dirPattern>(mcidas|viirs)/.*/.*/.*/.*</dirPattern>
<filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<displayLabel>{1}</displayLabel>
</dataSet>
</category>
<category>
<name>Sounding</name>
<name>Profiles</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(acarssounding|bufrua|goessounding|poessounding|profiler|raobs)</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</category>
<category>
<name>ModelSounding</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(modelsounding)/(.*)</dirPattern>
<dirPattern>(bufrmos)(.*)</dirPattern>
<displayLabel>{1} - {2}</displayLabel>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
<filePattern>.*(\d{4})-(\d{2})-(\d{2})[-_](\d{2}).*</filePattern>
<dataSet>
<dirPattern>(acarssounding|bufrua|goessounding|poessounding|profiler)</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</dataSet>
</category>
<category>
<!-- Find mosaic dataset and add here. -->
<name>radar</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>radar/(.*)/(.*)</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
<filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<dataSet>
<dirPattern>radar/(.*)/(.*)</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
<filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</dataSet>
</category>
</archive>

140
edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archiver/purger/RAW_DATA.xml
View file

@ -24,43 +24,60 @@
* Date Ticket# Engineer Description
* ============ ========== =========== ==========================
* Jun 20, 2013 1966 rferrel Initial creation
* Aug 05, 2012 2224 rferrel Changes to add dataSet tags.
*
* @author rferrel
* @version 1.0
-->
<!--
The <archive> contains five types of tags:
The <archive> contains four types of tags:
<name> - Required tag. The id for the archive such as Raw or Processed.
Used in the GUIs to filter which archive to display.
<rootDir> - Required tag. The root directory on the edex server for the archive.
<minRetentionHours> - Required tag. The default number of hours to retain data in the <rootDir>
<category> - Logical grouping of archive's sub-directories.
The <category> has six types of tags:
The <category> has four types of tags:
<name> - Required tag. The id for the category such as grib2 or redbook.
Used in the GUIs to filter what is the display in the table.
<extRetentionHours> - Optional tag. The extended retentionHours for selected directories of this category.
Default is the archive's <minRetentionHours>.
<dirPattern> - Required tag. A regex pattern for finding directories for this category.
<dataSet> - Required to have a least one. Each one contains a set of tags explained below.
<selectedDisplayNames> - A directory matching <dirPattern>. These are selected directories from the Retention GUI. The purger
will used the category's <extRetentionHours> instead of the Arhivie's <minRetentionHours>.
An optional tag; may have more then one.(NOTE these are set internally when a selection configuration
file is loaded. They should not appear in the configuration file.)
The <dataSet> contains ? types of tags:
<dirPattern> - Required to have at least one. A regex pattern for finding directories for this category.
The pattern is relative to the archive's <rootDir>. Wildcard patterns do not cross directory
delimiter /. Thus to match 3 levels of directories you would need .*/.*/.* .
See patterns and groups section. There may be more then one of these tags. The restriction is they
must all have the same number of groupings and be in the same order.
See patterns and groups section. There may be more then one of these tags in a <dataSet>. The
restriction is they must all have the same number of groupings and be in the same order to match up
with the <displayLabel>'s, and <dateGroupIndicies>'s values.
<filePattern> - Optional tag. A pattern to find files in the directories that match the <dirPattern>.
Default is everything in the directories that match <dirPattern>.
See patterns and groups section.
<displayLabel> - Required tag. The label to display for directories that match <dirPattern>. Any group in the
<dirPattern> may be made part of the label by placing the group's index inside parenthesis, {1}.
More then one directory may match the <dirPattern>. The archive GUIs may collapse them into a
More then one directory may match the <dirPattern>. The archive GUIs may collapse them into a
single table entry.
<dateGroupIndicies> - Required tag. A comma separated list of 4 numbers which are the index of the groups in
<dirPattern> and/or <filePattern> that contain in order the year, mouth, day and hour information.
<timeType> - Optional tag to determine what type of time stamp is being used to get files/directories for retention
and case creation. The value dictates how many groupings in the <dirPattern>s and/or <filePattern> are
used to get the time stamp for a file. The four values are:
Date - (default) the time stamp is made up of four groups in the patterns: year, month, day and hour.
EpochSec - The time stamp has one group in the patterns which is the epoch time in seconds.
EpochMS - The time stamp has one group in the patterns which is the epoch time in milliseconds.
File - No group is used to get the time stamp. Instead use the files date of last modification.
<dateGroupIndicies> - Required tag when <timeType> has any value but File.
Date - A comma separated list of 4 numbers which are in order the index for year, month, day and hour.
EpochSec - A number which is the index for the epoch in seconds.
EpochMS - A number which is the index for the epoch in milliseconds.
File - Not needed since no group is used to get the time stamp.
This is used to determine what files/directories to retain or a range of directories/files to copy
for case creation. Note to get the group's index the <dirPattern> and <filePattern> are combined.
Thus if there are 5 groups in the <dirPattern> then the first group in the <filePattern> is index 6.
<selectedDisplayNames> - A directory matching <dirPattern>. These are selected directories from the Retention GUI. The purger
will used the category's <extRetentionHours> instead of the Arhivie's <minRetentionHours>.
An optional tag; may have more then one.
Patterns and groups.
The <dirPattern> and <filePattern> use Java regex expressions; similar to the ldm's pqact.conf file.
For more details see http://docs.oracle.com/javase/tutorial/essential/regex/
@ -77,7 +94,7 @@
Finally the sixth group will match any sub-directory that in the hour directory.
Thus the directory paths <rootPath>/grib2/20130527/18/GFS will generate the display string, grib2 - GFS, and from the grouping we
can find the year, 201; month, 05; day, 27 and hour, 18.
can find the year, 2013; month, 05; day, 27 and hour, 18.
Example with <filePattern>:
<dirPattern>hdf5/(redbook)</dirPattern>
@ -89,11 +106,14 @@
<category>
<name>Observation</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(acars|airep|airmet|taf)</dirPattern>
<dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
<dataSet>
<dirPattern>(acars|airep|airmet|taf)</dirPattern>
<dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
<displayLabel>{1}</displayLabel>
<timeType>Date</timeType>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
</dataSet>
</category>
The first <dirPattern> looks for files matching the <filePattern> in the directories acars, airep, airmet or taf.
The second <dirPattern> expects to find the files in subdirectories of bufrsigwx or sfcobs such as bufrsigwx/SWH.
@ -109,54 +129,78 @@
<name>Raw</name>
<rootDir>/data_store/</rootDir>
<minRetentionHours>24</minRetentionHours>
<category>
<name>Local</name>
<extRetentionHours>168</extRetentionHours>
<dataSet>
<dirPattern>(manual)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</dataSet>
</category>
<category>
<name>Model</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(grib|grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<displayLabel>{1} - {6}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</category>
<category>
<name>ModelSounding</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(bufrmos|modelsounding)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</category>
<category>
<name>Misc</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(binlightning|climate|cwa|lsr|manual|misc_adm_messages|redbook|shef|summaries|svrwx|tcg|tcs|text|wwa|xml)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<dataSet>
<dirPattern>(grib|grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<displayLabel>{1} - {6}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</dataSet>
<dataSet>
<dirPattern>(bufrmos|modelsounding)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</dataSet>
</category>
<category>
<name>Observation</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(airep|airmet|bufrascat|bufrhdw|bufrmthdw|bufrncwf|bufrsigwx|bufrssmi|convsigmet|fire_wx_spot_fcst_reports|forecast|fssobs|intlsigmet|MAROB|maritime|metar|misc_sfc_obs|nonconvsigmet|pirep|sfcobs|synoptic)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<dirPattern>acars/(.*)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<dataSet>
<dirPattern>(airep|binlightning|bufrascat|bufrhdw|bufrmthdw|bufrssmi|lsr|MAROB|maritime|metar|misc_sfc_obs|pirep|sfcobs|shef|svrwx|synoptic)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</dataSet>
<dataSet>
<dirPattern>(airep|binlightning|bufrascat|bufrhdw|bufrmthdw|bufrssmi|idft|lsr|MAROB|maritime|metar|misc_sfc_obs|pirep|sfcobs|shef|svrwx|synoptic)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<dirPattern>(acars)/(.*)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1} - {2}</displayLabel>
<dateGroupIndices>3,4,5,6</dateGroupIndices>
</dataSet>
</category>
<category>
<name>Products</name>
<extRetentionHours>168</extRetentionHours>
<dataSet>
<dirPattern>(airmet|bufrncwf|bufrsigwx|climate|convsigmet|cwa|fire_wx_spot_fcst_reports|forecast|intlsigmet|misc_adm_messages|nonconvsigmet|redbook|summaries|tcg|tcs|text|wwa|xml)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</dataSet>
</category>
<category>
<name>radar</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>radar/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<displayLabel>{5}</displayLabel>
<dateGroupIndices>1,2,3,4</dateGroupIndices>
<dataSet>
<dirPattern>radar/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<displayLabel>{5}</displayLabel>
<dateGroupIndices>1,2,3,4</dateGroupIndices>
</dataSet>
</category>
<category>
<name>Satellite</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>sat/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<displayLabel>{5}</displayLabel>
<dateGroupIndices>1,2,3,4</dateGroupIndices>
<dataSet>
<dirPattern>sat/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
<displayLabel>{5}</displayLabel>
<dateGroupIndices>1,2,3,4</dateGroupIndices>
</dataSet>
</category>
<category>
<name>Sounding</name>
<name>Profiles</name>
<extRetentionHours>168</extRetentionHours>
<dirPattern>(acarssounding|bufrua|goessounding|poessounding|profiler|raobs|upperair)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
<dataSet>
<dirPattern>(bufrua|goessounding|poessounding|profiler|raobs|upperair)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
<displayLabel>{1}</displayLabel>
<dateGroupIndices>2,3,4,5</dateGroupIndices>
</dataSet>
</category>
</archive>