From 8884343d89e439d4df54ff012e8ac8d61b072f8a Mon Sep 17 00:00:00 2001 From: Roger Ferrel Date: Tue, 6 Aug 2013 15:44:45 -0500 Subject: [PATCH] Issue #2224 Changes to have DataSet in configuration. Change-Id: If626c4ef5b1da5bf5b6c660a0337009620a04db8 Former-commit-id: 705a3482f83fb561442c2d9c5fc7cc7177ec4d73 [formerly 1863d4d65bb8e6db9565e6a050b3d70feaddd6f6] [formerly 3ecf95b18a803707eda249eaa506712ac5143a16 [formerly de48fa389414c87ba557b70a480ce6b45144f4aa]] Former-commit-id: 3ecf95b18a803707eda249eaa506712ac5143a16 Former-commit-id: 3d9be94e7efa2ef86b9b65f0f5600585cba5118b --- .../raytheon/uf/viz/archive/data/SizeJob.java | 1 + .../archive/config/ArchiveConfigManager.java | 219 +++++++------- .../common/archive/config/CategoryConfig.java | 165 ++--------- .../archive/config/CategoryDataSet.java | 280 ++++++++++++++++++ .../config/CategoryFileDateHelper.java | 119 ++++---- .../uf/common/archive/config/DisplayData.java | 25 +- .../base/archiver/purger/PROCESSED_DATA.xml | 182 ++++++++---- .../base/archiver/purger/RAW_DATA.xml | 140 ++++++--- 8 files changed, 694 insertions(+), 437 deletions(-) create mode 100644 edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryDataSet.java diff --git a/cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/data/SizeJob.java b/cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/data/SizeJob.java index c672cfbba5..fe2799316d 100644 --- a/cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/data/SizeJob.java +++ b/cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/data/SizeJob.java @@ -40,6 +40,7 @@ import com.raytheon.uf.common.time.util.TimeUtil; * ------------ ---------- ----------- -------------------------- * Jun 13, 2013 rferrel Initial creation * Jul 24, 2013 #2220 rferrel Change to get all data sizes only one time. + * Aug 02, 2013 #2224 rferrel Changes for new configuration files. * Aug 06, 2013 #2222 rferrel Changes to display all selected data. * * diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java index 48a2b3a2aa..73dc40d5c6 100644 --- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java @@ -32,10 +32,11 @@ import java.util.Calendar; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.TimeZone; -import java.util.TreeSet; +import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -85,6 +86,7 @@ import com.raytheon.uf.common.util.FileUtil; * Changed to use File.delete() instead of Apache FileUtil.deleteQuietly(). * Added warn logging for failure to delete. * Jul 24, 2013 2221 rferrel Changes for select configuration. + * Aug 06, 2013 2224 rferrel Changes to use DataSet. * * * @@ -529,91 +531,79 @@ public class ArchiveConfigManager { */ private List getDisplayFiles(DisplayData displayData, long startTime, long endTime) { + List fileList = new LinkedList(); ArchiveConfig archiveConfig = displayData.archiveConfig; - CategoryConfig categoryConfig = displayData.categoryConfig; - String[] indexValues = categoryConfig.getDateGroupIndices().split( - "\\s*,\\s*"); - int yearIndex = Integer.parseInt(indexValues[0]); - int monthIndex = Integer.parseInt(indexValues[1]); - int dayIndex = Integer.parseInt(indexValues[2]); - int hourIndex = Integer.parseInt(indexValues[3]); + for (CategoryDataSet dataSet : displayData.dataSets) { - String filePatternStr = categoryConfig.getFilePattern(); + int[] timeIndices = dataSet.getTimeIndices(); - boolean dirOnly = (filePatternStr == null) - || ".*".equals(filePatternStr); + String filePatternStr = dataSet.getFilePattern(); - List dirs = displayData.dirs; + boolean dirOnly = dataSet.isDirOnly(); - int beginIndex = archiveConfig.getRootDir().length(); + List dirs = displayData.dirsMap.get(dataSet); - Calendar fileCal = TimeUtil.newCalendar(); - fileCal.setTimeZone(TimeZone.getTimeZone("UTC")); + int beginIndex = archiveConfig.getRootDir().length(); - List fileList = new ArrayList(); + Calendar fileCal = TimeUtil.newCalendar(); + fileCal.setTimeZone(TimeZone.getTimeZone("UTC")); - if (dirOnly) { - for (String dirPattern : categoryConfig.getDirPatternList()) { - Pattern pattern = Pattern.compile(dirPattern); + if (dirOnly) { + for (String dirPattern : dataSet.getDirPatterns()) { + Pattern pattern = dataSet.getPattern(dirPattern); - for (File dir : dirs) { - String path = dir.getAbsolutePath().substring(beginIndex); - Matcher matcher = pattern.matcher(path); - if (matcher.matches()) { - int year = Integer.parseInt(matcher.group(yearIndex)); - // Adjust month value to Calendar's 0 - 11 - int month = Integer.parseInt(matcher.group(monthIndex)) - 1; - int day = Integer.parseInt(matcher.group(dayIndex)); - int hour = Integer.parseInt(matcher.group(hourIndex)); - fileCal.set(year, month, day, hour, 0, 0); - long fileTime = fileCal.getTimeInMillis(); - if ((startTime <= fileTime) && (fileTime < endTime)) { - fileList.add(dir); - } - } - } - } - } else { - for (String dirPattern : categoryConfig.getDirPatternList()) { - Pattern pattern = Pattern.compile(dirPattern + File.separator - + categoryConfig.getFilePattern()); - final Pattern filePattern = Pattern.compile("^" - + filePatternStr + "$"); - for (File dir : dirs) { - List fList = FileUtil.listDirFiles(dir, - new FileFilter() { - - @Override - public boolean accept(File pathname) { - return filePattern.matcher( - pathname.getName()).matches(); - } - }, false); - for (File file : fList) { - String path = file.getAbsolutePath().substring( + for (File dir : dirs) { + String path = dir.getAbsolutePath().substring( beginIndex); Matcher matcher = pattern.matcher(path); if (matcher.matches()) { - int year = Integer.parseInt(matcher - .group(yearIndex)); - // Adjust month value to Calendar's 0 - 11 - int month = Integer.parseInt(matcher - .group(monthIndex)) - 1; - int day = Integer.parseInt(matcher.group(dayIndex)); - int hour = Integer.parseInt(matcher - .group(hourIndex)); - fileCal.set(year, month, day, hour, 0, 0); - long fileTime = fileCal.getTimeInMillis(); + Long fileTime = dataSet.getMatchTimeInMilliseconds( + timeIndices, matcher); + if (fileTime == null) { + fileTime = dir.lastModified(); + } if ((startTime <= fileTime) && (fileTime < endTime)) { - fileList.add(file); + fileList.add(dir); + } + } + } + } + } else { + for (String dirPattern : dataSet.getDirPatterns()) { + Pattern pattern = dataSet.getPattern(dirPattern); + final Pattern filePattern = Pattern.compile("^" + + filePatternStr + "$"); + for (File dir : dirs) { + List fList = FileUtil.listDirFiles(dir, + new FileFilter() { + + @Override + public boolean accept(File pathname) { + return filePattern.matcher( + pathname.getName()).matches(); + } + }, false); + for (File file : fList) { + String path = file.getAbsolutePath().substring( + beginIndex); + Matcher matcher = pattern.matcher(path); + if (matcher.matches()) { + Long timestamp = dataSet + .getMatchTimeInMilliseconds( + timeIndices, matcher); + long fileTime = timestamp == null ? file + .lastModified() : timestamp.longValue(); + if ((startTime <= fileTime) + && (fileTime < endTime)) { + fileList.add(file); + } } } } } } } - return fileList; } @@ -625,15 +615,13 @@ public class ArchiveConfigManager { * @param categoryConfig * @return dirs */ - private List getDirs(ArchiveConfig archiveConfig, - CategoryConfig categoryConfig) { + private List getDirs(File rootFile, CategoryDataSet dataSet) { List resultDirs = new ArrayList(); - File rootFile = new File(archiveConfig.getRootDir()); List dirs = new ArrayList(); List tmpDirs = new ArrayList(); List swpDirs = null; - for (String dirPattern : categoryConfig.getDirPatternList()) { + for (String dirPattern : dataSet.getDirPatterns()) { String[] subExpr = dirPattern.split(File.separator); dirs.clear(); dirs.add(rootFile); @@ -679,50 +667,59 @@ public class ArchiveConfigManager { Map> displayMap = new HashMap>(); ArchiveConfig archiveConfig = archiveMap.get(archiveName); + String rootDirName = archiveConfig.getRootDir(); CategoryConfig categoryConfig = findCategory(archiveConfig, categoryName); - List dirPatternList = categoryConfig.getDirPatternList(); + File rootFile = new File(rootDirName); + TreeMap displays = new TreeMap(); + for (CategoryDataSet dataSet : categoryConfig.getDataSetList()) { + List dataSetDirPatterns = dataSet.getDirPatterns(); - // index for making directory paths' relative to the root path. - List dirs = getDirs(archiveConfig, categoryConfig); + List dirs = getDirs(rootFile, dataSet); - File rootFile = new File(archiveMap.get(archiveName).getRootDir()); - int beginIndex = rootFile.getAbsolutePath().length() + 1; - List patterns = new ArrayList(dirPatternList.size()); + int beginIndex = rootFile.getAbsolutePath().length() + 1; + List patterns = new ArrayList( + dataSetDirPatterns.size()); - for (String dirPattern : dirPatternList) { - Pattern pattern = Pattern.compile("^" + dirPattern + "$"); - patterns.add(pattern); - } + for (String dirPattern : dataSetDirPatterns) { + Pattern pattern = Pattern.compile("^" + dirPattern + "$"); + patterns.add(pattern); + } - TreeSet displays = new TreeSet( - String.CASE_INSENSITIVE_ORDER); + MessageFormat msgfmt = new MessageFormat(dataSet.getDisplayLabel()); + StringBuffer sb = new StringBuffer(); + FieldPosition pos0 = new FieldPosition(0); - MessageFormat msgfmt = new MessageFormat(categoryConfig.getDisplay()); - StringBuffer sb = new StringBuffer(); - FieldPosition pos0 = new FieldPosition(0); - - for (File dir : dirs) { - String path = dir.getAbsolutePath().substring(beginIndex); - for (Pattern pattern : patterns) { - Matcher matcher = pattern.matcher(path); - if (matcher.matches()) { - sb.setLength(0); - String[] args = new String[matcher.groupCount() + 1]; - args[0] = matcher.group(); - for (int i = 1; i < args.length; ++i) { - args[i] = matcher.group(i); + for (File dir : dirs) { + String path = dir.getAbsolutePath().substring(beginIndex); + for (Pattern pattern : patterns) { + Matcher matcher = pattern.matcher(path); + if (matcher.matches()) { + sb.setLength(0); + String[] args = new String[matcher.groupCount() + 1]; + args[0] = matcher.group(); + for (int i = 1; i < args.length; ++i) { + args[i] = matcher.group(i); + } + String displayLabel = msgfmt.format(args, sb, pos0) + .toString(); + List displayDirs = displayMap.get(displayLabel); + if (displayDirs == null) { + displayDirs = new ArrayList(); + displayMap.put(displayLabel, displayDirs); + } + displayDirs.add(dir); + DisplayData displayData = displays.get(displayLabel); + if (displayData == null) { + displayData = new DisplayData(archiveConfig, + categoryConfig, dataSet, displayLabel); + displays.put(displayLabel, displayData); + } else { + displayData.dataSets.add(dataSet); + } + displayData.dirsMap.put(dataSet, displayDirs); + break; } - String displayLabel = msgfmt.format(args, sb, pos0) - .toString(); - List displayDirs = displayMap.get(displayLabel); - if (displayDirs == null) { - displayDirs = new ArrayList(); - displayMap.put(displayLabel, displayDirs); - } - displayDirs.add(dir); - displays.add(displayLabel); - break; } } } @@ -730,15 +727,7 @@ public class ArchiveConfigManager { List displayDataList = new ArrayList( displays.size()); - for (String displayLabel : displays) { - DisplayData displayData = new DisplayData(archiveConfig, - categoryConfig, displayLabel, displayMap.get(displayLabel)); - if (setSelect) { - displayData.setSelected(categoryConfig - .getSelectedDisplayNames().contains(displayLabel)); - } - displayDataList.add(displayData); - } + displayDataList.addAll(displays.values()); return displayDataList; } diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java index a97dcf22d3..42f5ba8d9a 100644 --- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java @@ -39,10 +39,13 @@ import javax.xml.bind.annotation.XmlRootElement; * <name>redbook</name> * <!-- When 0 default to the parent archive's retentionHours --> * <retentionHours>0</retentionHours> - * <dirPattern>hdf5/(redbook)</dirPattern> - * <displayLabel>{1}</displayLabel> - * <filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern> - * <dateGroupIndices>2,3,4,5</dateGroupIndices> + * <dataSet> + * <dirPattern>hdf5/(redbook)</dirPattern> + * <displayLabel>{1}</displayLabel> + * <filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern> + * <timeType>Date</timeType> + * <dateGroupIndices>2,3,4,5</dateGroupIndices> + * </dataSet> * </category> * * @@ -52,9 +55,12 @@ import javax.xml.bind.annotation.XmlRootElement; * <category> * <name>Model grib</name> * <retentionHours>0</retentionHours> - * <dirPattern>grib/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern> - * <displayLabel>{5}</displayLabel> - * <dateGroupIndices>1,2,3,4</dateGroupIndices> + * <dataSet> + * <dirPattern>grib/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern> + * <displayLabel>{5}</displayLabel> + * <timeType>Date</timeType> + * <dateGroupIndices>1,2,3,4</dateGroupIndices> + * </dataSet> * </category> * * @@ -65,6 +71,7 @@ import javax.xml.bind.annotation.XmlRootElement; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * May 1, 2013 1966 rferrel Initial creation + * Aug 03, 2013 2224 rferrel Changes to include DataSet. * * * @@ -88,63 +95,8 @@ public class CategoryConfig implements Comparable { @XmlElement(name = "extRetentionHours") private int retentionHours; - /** - * A regex pattern to find directories controlled by the category. These - * directories should be relative to the parent archive's directory. For - * example: - * - * 
-     * <dirPattern>grib2/\d{8}/\d{2}/(.*)/</dirPattern>
-     *
- */ - @XmlElement(name = "dirPattern") - private List dirPatternList; - - /** - * Use to display the information found by the dirPattern. Any groups in the - * dirPatern may be displayed. For example: - * - * 
-     * <dirName>(grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirName>
-     * <displayLabel>{1} - {6}</displayLabel>
-     *
- * - * The {1} will be replaced by the first group (grib2) in the regex - * expression in dirName. The {6} is the sixth group (.*). {0} is the whole - * expression match. - */ - @XmlElement(name = "displayLabel") - private String display; - - /** - * A comma separated list of 4 numbers representing the group indices - * specifying the location of the numeric date time stamp. The indices must - * be in the order of year, month, day and hour. The group numbering starts - * with the first group in the dirPattern and continues with any grouping in - * the filePattern. - * - * 
-     *   <dirPattern>hdf5/(redbook)</dirPattern>
-     *   <displayLabel>{1}</displayLabel>
-     *   <filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
-     *   <dateGroupIndices>2,3,4,5</dateGroupIndices>
-     *
- */ - @XmlElement(name = "dateGroupIndices") - private String dateGroupIndices; - - /** - * A saveDir directory may contain files with data for several days. This - * allows any of the year, month, day and hour to be part of a file name. - * Default is all files in the directory. For example: - * - * 
-     * <saveDir>hd5/redbook/(^/]*//)</saveDir>
-     * <saveFile>redbook-${YYYY}-${MM}-${DD}-${HH}\..*<saveFiles>
-     *
- */ - @XmlElement(name = "filePattern") - private String filePattern; + @XmlElement(name = "dataSet") + private List dataSetList; @XmlElement(name = "selectedDisplayName") private final Collection selectedDisplayNames = new TreeSet(); @@ -188,76 +140,12 @@ public class CategoryConfig implements Comparable { this.retentionHours = retentionHours; } - /** - * Obtain the list of directory patterns. - * - * @return dirPatternList - */ - public List getDirPatternList() { - return new ArrayList(dirPatternList); + public List getDataSetList() { + return new ArrayList(dataSetList); } - /** - * Set the directory pattern list; must not be null. - * - * @param dirPatternList - */ - public void setDirPatternList(List dirPatternList) { - this.dirPatternList = dirPatternList; - } - - /** - * Get the display label pattern. - * - * @return display - */ - public String getDisplay() { - return display == null ? "" : display; - } - - /** - * Set the display label pattern. - * - * @param display - */ - public void setDisplay(String display) { - this.display = display; - } - - /** - * Get the save directory pattern.. - * - * @return dateGroups - */ - public String getDateGroupIndices() { - return dateGroupIndices; - } - - /** - * Set the save directory pattern; must not be null. - * - * @param saveDir - */ - public void setDateGroupIndices(String dateGroupIndices) { - this.dateGroupIndices = dateGroupIndices; - } - - /** - * Get the save files pattern. - * - * @return saveFiles - */ - public String getFilePattern() { - return filePattern; - } - - /** - * Set the save files pattern; may be null. - * - * @param saveFiles - */ - public void setFilePattern(String filePattern) { - this.filePattern = filePattern; + public void setDataSetList(List dataSetList) { + this.dataSetList = dataSetList; } public Collection getSelectedDisplayNames() { @@ -285,9 +173,9 @@ public class CategoryConfig implements Comparable { */ @Override public int compareTo(CategoryConfig o) { - return getDisplay().compareToIgnoreCase(o.getDisplay()); + return getName().compareToIgnoreCase(o.getName()); } - + /* * (non-Javadoc) * @@ -298,13 +186,10 @@ public class CategoryConfig implements Comparable { StringBuilder sb = new StringBuilder(); sb.append("Category [ name: ").append(getName()); sb.append(", retentionHours: ").append(getRetentionHours()); - sb.append(", dirPatternList[ "); - for (String dirPattern : getDirPatternList()) { - sb.append(" \"").append(dirPattern).append("\","); + sb.append(", dataSetList[ "); + for (CategoryDataSet dataSet : getDataSetList()) { + sb.append(dataSet).append(", "); } - sb.append("], filePattern: ").append(getFilePattern()); - sb.append(", displayLabel: ").append(getDisplay()); - sb.append(", dateGroupIndices: ").append(getDateGroupIndices()); sb.append(", selectedDisplayNames: "); if (selectedDisplayNames == null) { sb.append("null"); diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryDataSet.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryDataSet.java new file mode 100644 index 0000000000..4a68ab7714 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryDataSet.java @@ -0,0 +1,280 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.archive.config; + +import java.io.File; +import java.util.Calendar; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import com.raytheon.uf.common.time.util.TimeUtil; + +/** + * A grouping of data set information used in a category. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 6, 2013  #2224      rferrel     Initial creation
+ * 
+ *
+ * + * @author rferrel + * @version 1.0 + */ +@XmlAccessorType(XmlAccessType.NONE) +@XmlRootElement(name = "dataSet") +public class CategoryDataSet { + public static final int YEAR_INDEX = 0; + + public static final int MONTH_INDEX = 1; + + public static final int DAY_INDEX = 2; + + public static final int HOUR_INDEX = 3; + + public static final int TIMESTAMP_INDEX = 0; + + /** + * Types of times and the number of indices for getting the time stamp from + * patterns. + */ + public static enum TimeType { + Date(4), EpochSec(1), EpochMS(1), File(0); + + private final int numIndices; + + private TimeType(int numIndices) { + this.numIndices = numIndices; + } + + public int getNumGroupIndices() { + return numIndices; + } + } + + /** + * List of directory patterns. + */ + @XmlElement(name = "dirPattern") + private List dirPatterns; + + /** + * Optional file pattern. + */ + @XmlElement(name = "filePattern") + private String filePattern; + + @XmlElement(name = "timeType") + private TimeType timeType; + + /** + * The display label. + */ + @XmlElement(name = "displayLabel") + private String displayLabel; + + private int[] timeIndices = null; + + /** + * The index with group number for getting desired parts of the TimeType. + */ + @XmlElement(name = "dateGroupIndices") + private String dateGroupIndices = ""; + + public List getDirPatterns() { + return dirPatterns; + } + + public void setDirPatterns(List dirPatterns) { + this.dirPatterns = dirPatterns; + } + + public String getFilePattern() { + return filePattern; + } + + public void setFilePattern(String filePattern) { + this.filePattern = filePattern; + } + + public TimeType getTimeType() { + return timeType; + } + + public void setTimeType(TimeType timeType) { + this.timeType = timeType; + } + + public String getDisplayLabel() { + return displayLabel; + } + + public void setDisplayLabel(String displayLabel) { + this.displayLabel = displayLabel; + } + + public String getDateGroupIndices() { + return dateGroupIndices; + } + + public void setDateGroupIndices(String dateGroupIndices) { + this.dateGroupIndices = dateGroupIndices; + this.timeIndices = null; + } + + /** + * Get the array of time indices based on time type and date group indices. + * + * @return timeIndices + */ + public int[] getTimeIndices() { + if (timeIndices == null) { + timeIndices = new int[timeType.getNumGroupIndices()]; + if (timeIndices.length > 0) { + String[] indexValues = getDateGroupIndices().split("\\s*,\\s*"); + for (int index = 0; index < timeIndices.length; ++index) { + timeIndices[index] = Integer.parseInt(indexValues[index]); + } + } + } + return timeIndices; + } + + /** + * Get the Pattern for dirPattern. + * + * @param dirPattern + * + * @return pattern or null if dirPattern not in the list of directory + * patterns. + */ + public Pattern getPattern(String dirPattern) { + Pattern pattern = null; + if (dirPatterns.contains(dirPattern)) { + if (isDirOnly()) { + pattern = Pattern.compile(dirPattern); + } else { + pattern = Pattern.compile(dirPattern + File.separator + + getFilePattern()); + } + } + return pattern; + } + + /** + * + * @return true when only the dirPatterns should be used. + */ + public boolean isDirOnly() { + return filePattern == null || filePattern.equals(".*"); + } + + /** + * Get time stamp for file based on timetype. Assumes file path matches + * + * @param timeIndices + * @param matcher + * @param file + * @return + */ + public Long getMatchTimeInMilliseconds(int[] timeIndices, Matcher matcher) { + return CategoryDataSet.getMatchTimeInMilliseconds(timeType, + timeIndices, matcher); + } + + /** + * Get file time based on time type. + * + * @param timeType + * @param timeIndices + * @param matcher + * @param file + * @return fileTime + */ + public static Long getMatchTimeInMilliseconds( + CategoryDataSet.TimeType timeType, int[] timeIndices, + Matcher matcher) { + Long fileTime = null; + switch (timeType) { + case Date: + Calendar fileCal = TimeUtil.newGmtCalendar(); + int year = Integer.parseInt(matcher + .group(timeIndices[CategoryDataSet.YEAR_INDEX])); + // Adjust month value to Calendar's 0 - 11 + int month = Integer.parseInt(matcher + .group(timeIndices[CategoryDataSet.MONTH_INDEX])) - 1; + int day = Integer.parseInt(matcher + .group(timeIndices[CategoryDataSet.DAY_INDEX])); + int hour = Integer.parseInt(matcher + .group(timeIndices[CategoryDataSet.HOUR_INDEX])); + fileCal.set(year, month, day, hour, 0, 0); + fileTime = fileCal.getTimeInMillis(); + break; + case EpochMS: + fileTime = Long.parseLong(matcher + .group(timeIndices[CategoryDataSet.TIMESTAMP_INDEX])); + break; + case EpochSec: + fileTime = Long.parseLong(matcher + .group(timeIndices[CategoryDataSet.TIMESTAMP_INDEX])); + fileTime *= TimeUtil.MILLIS_PER_SECOND; + break; + case File: + fileTime = null; + break; + default: + fileTime = null; + break; + } + return fileTime; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder("DataSet[ "); + sb.append("TimeType: ").append(getTimeType()); + sb.append("dateGroupIndices: ").append(getDateGroupIndices()); + sb.append(", isDirOnly: ").append(isDirOnly()); + sb.append(", displayLabel: ").append(getDisplayLabel()); + sb.append(", dirPatterns[ "); + for (String dirPattern : getDirPatterns()) { + sb.append(dirPattern).append(", "); + } + sb.append("], filePattern: ").append( + filePattern == null ? "null" : filePattern); + sb.append("]"); + return sb.toString(); + } +} diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java index 4534160878..7cd3030a51 100644 --- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java @@ -40,6 +40,7 @@ import com.raytheon.uf.common.time.util.TimeUtil; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jun 21, 2013 1965 bgonzale Initial creation + * Aug 03, 2013 2224 rferrel Changes for new configuration files. * * * @@ -56,13 +57,11 @@ public class CategoryFileDateHelper implements IFileDateHelper { private final Pattern categoryTopLevelDirPattern; - private final int yearIndex; + private final CategoryDataSet.TimeType timeType; - private final int monthIndex; + private final boolean isDirOnly; - private final int dayIndex; - - private final int hourIndex; + private final int[] timeIndices; /** * Initialization constructor. @@ -76,13 +75,13 @@ public class CategoryFileDateHelper implements IFileDateHelper { */ public CategoryDateInfo(Pattern datePattern, Pattern categoryTopLevelDirPattern, - int yearIndex, int monthIndex, int dayIndex, int hourIndex) { + CategoryDataSet.TimeType timeType, boolean isDirOnly, + int[] timeIndices) { this.datePattern = datePattern; this.categoryTopLevelDirPattern = categoryTopLevelDirPattern; - this.yearIndex = yearIndex; - this.monthIndex = monthIndex; - this.dayIndex = dayIndex; - this.hourIndex = hourIndex; + this.timeType = timeType; + this.isDirOnly = isDirOnly; + this.timeIndices = timeIndices; } } @@ -91,8 +90,6 @@ public class CategoryFileDateHelper implements IFileDateHelper { private final String rootDir; - private final boolean isDirOnly; - /** * Initialization constructor. * @@ -101,38 +98,37 @@ public class CategoryFileDateHelper implements IFileDateHelper { * categoryTopLevelDirPattern */ public CategoryFileDateHelper(CategoryConfig config, String rootDir) { - List categoryDirPatternList = config.getDirPatternList(); - this.dateInfoList = new ArrayList( - categoryDirPatternList.size()); - - String filePatternStr = config.getFilePattern(); this.rootDir = rootDir; - this.isDirOnly = (filePatternStr == null) - || ".*".equals(filePatternStr); + List categoryDataSetList = config.getDataSetList(); + int size = 0; + for (CategoryDataSet dataSet : categoryDataSetList) { + size += dataSet.getDirPatterns().size(); + } - for (String patternString : categoryDirPatternList) { - Pattern datePattern = null; - if (isDirOnly) { - datePattern = Pattern.compile(patternString); - } else { - datePattern = Pattern.compile(patternString + File.separator - + config.getFilePattern()); + this.dateInfoList = new ArrayList( + size); + + boolean isDirOnly; + CategoryDataSet.TimeType timeType; + for (CategoryDataSet dataSet : categoryDataSetList) { + isDirOnly = dataSet.isDirOnly(); + timeType = dataSet.getTimeType(); + + for (String patternString : dataSet.getDirPatterns()) { + Pattern datePattern = dataSet.getPattern(patternString); + int dirSeparatorIndex = patternString + .indexOf(File.separatorChar); + patternString = dirSeparatorIndex > patternString.length() + || dirSeparatorIndex < 0 ? patternString + : patternString.substring(0, dirSeparatorIndex); + Pattern categoryTopLevelDirPattern = Pattern + .compile(patternString); + int[] timeIndices = dataSet.getTimeIndices(); + + dateInfoList.add(new CategoryDateInfo(datePattern, + categoryTopLevelDirPattern, timeType, isDirOnly, + timeIndices)); } - int dirSeparatorIndex = patternString.indexOf(File.separatorChar); - patternString = dirSeparatorIndex > patternString.length() - || dirSeparatorIndex < 0 ? patternString : patternString - .substring(0, dirSeparatorIndex); - Pattern categoryTopLevelDirPattern = Pattern.compile(patternString); - String[] indexValues = config.getDateGroupIndices().split( - "\\s*,\\s*"); - int yearIndex = Integer.parseInt(indexValues[0]); - int monthIndex = Integer.parseInt(indexValues[1]); - int dayIndex = Integer.parseInt(indexValues[2]); - int hourIndex = Integer.parseInt(indexValues[3]); - - dateInfoList.add(new CategoryDateInfo(datePattern, - categoryTopLevelDirPattern, yearIndex, monthIndex, - dayIndex, hourIndex)); } } @@ -145,36 +141,39 @@ public class CategoryFileDateHelper implements IFileDateHelper { */ @Override public Calendar getFileDate(String filenamePath) { - String pathForPatternCheck = filenamePath.substring(rootDir.length()); - pathForPatternCheck = isDirOnly ? FilenameUtils - .getFullPathNoEndSeparator(pathForPatternCheck) - : pathForPatternCheck; + String pathForFilePatternCheck = filenamePath.substring(rootDir + .length()); + String pathForDirPatternCheck = FilenameUtils + .getFullPathNoEndSeparator(pathForFilePatternCheck); Calendar result = null; + Long timestamp = null; for (CategoryDateInfo dateInfo : dateInfoList) { - Matcher matcher = dateInfo.datePattern.matcher(pathForPatternCheck); + Matcher matcher = null; + if (dateInfo.isDirOnly) { + matcher = dateInfo.datePattern.matcher(pathForDirPatternCheck); + } else { + matcher = dateInfo.datePattern.matcher(pathForFilePatternCheck); + } if (matcher.matches()) { - int year = Integer.parseInt(matcher.group(dateInfo.yearIndex)); - // Adjust month value to Calendar's 0 - 11 - int month = Integer - .parseInt(matcher.group(dateInfo.monthIndex)) - 1; - int day = Integer.parseInt(matcher.group(dateInfo.dayIndex)); - int hour = Integer.parseInt(matcher.group(dateInfo.hourIndex)); - - result = TimeUtil.newGmtCalendar(); - result.set(year, month, day, hour, 0, 0); + timestamp = CategoryDataSet.getMatchTimeInMilliseconds( + dateInfo.timeType, dateInfo.timeIndices, matcher); break; } } - if (result == null) { + + if (timestamp == null) { // no matching pattern, use file last modified date File file = new File(filenamePath); - long lastModifiedMillis = file.lastModified(); - - result = TimeUtil.newGmtCalendar(); - result.setTimeInMillis(lastModifiedMillis); + timestamp = file.lastModified(); } + + // TODO future speed improvement refactor IFileDateHelper to have a + // method that returns a long instead of Calendar. That will prevent + // converting Calendar to long then back to a Calendar. + result = TimeUtil.newGmtCalendar(); + result.setTimeInMillis(timestamp); return result; } diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/DisplayData.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/DisplayData.java index 40a66100e3..e5cb689683 100644 --- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/DisplayData.java +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/DisplayData.java @@ -1,8 +1,11 @@ package com.raytheon.uf.common.archive.config; import java.io.File; +import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import com.raytheon.uf.common.util.SizeUtil; @@ -18,6 +21,7 @@ import com.raytheon.uf.common.util.SizeUtil; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jun 7, 2013 1966 rferrel Initial creation + * Aug 02, 2013 2224 rferrel Changes to include DataSet in configuration. * Aug 06, 2013 2222 rferrel Changes to display all selected data. * * @@ -71,22 +75,26 @@ public class DisplayData implements Comparable { /** The data's category configuration. */ protected final CategoryConfig categoryConfig; + protected final List dataSets = new ArrayList(); + /** The display label for this data. */ protected final String displayLabel; /** - * List of directories for the display label matching the category's - * directory pattern and found under the archive's root directory. + * Mappings of a list of directories for the display label matching the data + * set's directory patterns and found under the archive's root directory. */ - protected final List dirs; + protected final Map> dirsMap = new HashMap>(); /** - * For use by GUI to indicate. Use to indicate selected for retention or for - * placing in a case. + * For use by GUI to indicate display label's row is selected. */ private boolean selected = false; - /** For use by GUI for indicating the size of the directories' contents. */ + /** + * For use by GUI for indicating the size of the display label's row + * contents. + */ private long size = UNKNOWN_SIZE; /** @@ -94,15 +102,14 @@ public class DisplayData implements Comparable { * * @param archiveConfig * @param categoryConfig + * @param dataSet * @param displayLabel - * @param dirs */ public DisplayData(ArchiveConfig archiveConfig, - CategoryConfig categoryConfig, String displayLabel, List dirs) { + CategoryConfig categoryConfig, CategoryDataSet dataSet, String displayLabel) { this.archiveConfig = archiveConfig; this.categoryConfig = categoryConfig; this.displayLabel = displayLabel; - this.dirs = dirs; } /** diff --git a/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archiver/purger/PROCESSED_DATA.xml b/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archiver/purger/PROCESSED_DATA.xml index 7ec34bdc8a..48b1d5d0d8 100644 --- a/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archiver/purger/PROCESSED_DATA.xml +++ b/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archiver/purger/PROCESSED_DATA.xml @@ -24,43 +24,60 @@ * Date Ticket# Engineer Description * ============ ========== =========== ========================== * Jun 20, 2013 1966 rferrel Initial creation + * Aug 05, 2012 2224 rferrel Changes to add dataSet tags. * * @author rferrel * @version 1.0 --> + (mcidas|viirs)/.*/.*/.*/.* + .*-(\d{4})-(\d{2})-(\d{2})-(\d{2}).* + 2,3,4,5 + {1} + - Sounding + Profiles 168 - (acarssounding|bufrua|goessounding|poessounding|profiler|raobs) - {1} - 2,3,4,5 - [^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..* - - - ModelSounding - 168 - (modelsounding)/(.*) - (bufrmos)(.*) - {1} - {2} - 3,4,5,6 - .*(\d{4})-(\d{2})-(\d{2})[-_](\d{2}).* + + (acarssounding|bufrua|goessounding|poessounding|profiler) + {1} + 2,3,4,5 + [^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..* + + radar 168 - radar/(.*)/(.*) - {1} - 3,4,5,6 - .*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..* + + radar/(.*)/(.*) + {1} + 3,4,5,6 + .*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..* + diff --git a/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archiver/purger/RAW_DATA.xml b/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archiver/purger/RAW_DATA.xml index 88308bd6ed..945e0731d6 100644 --- a/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archiver/purger/RAW_DATA.xml +++ b/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archiver/purger/RAW_DATA.xml @@ -24,43 +24,60 @@ * Date Ticket# Engineer Description * ============ ========== =========== ========================== * Jun 20, 2013 1966 rferrel Initial creation + * Aug 05, 2012 2224 rferrel Changes to add dataSet tags. * * @author rferrel * @version 1.0 -->