Merge "Issue #2224 Changes to have DataSet in configuration." into development

Former-commit-id: 5548ae76b1 [formerly d8124958b5] [formerly 2de533cbbf [formerly d4a46b8283ac0cb47f62c43b999d7f119161e08d]]
Former-commit-id: 2de533cbbf
Former-commit-id: 4605595354

cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/data/SizeJob.java

@@ -40,6 +40,7 @@ import com.raytheon.uf.common.time.util.TimeUtil;
  * ------------ ---------- ----------- --------------------------
  * Jun 13, 2013            rferrel     Initial creation
  * Jul 24, 2013 #2220      rferrel     Change to get all data sizes only one time.
+ * Aug 02, 2013 #2224      rferrel     Changes for new configuration files.
  * Aug 06, 2013 #2222      rferrel     Changes to display all selected data.
  * 
  * </pre>

edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java

@@ -32,10 +32,11 @@ import java.util.Calendar;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
-import java.util.TreeSet;
+import java.util.TreeMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -85,6 +86,7 @@ import com.raytheon.uf.common.util.FileUtil;
  *                                     Changed to use File.delete() instead of Apache FileUtil.deleteQuietly().
  *                                     Added warn logging for failure to delete.
  * Jul 24, 2013 2221       rferrel     Changes for select configuration.
+ * Aug 06, 2013 2224       rferrel     Changes to use DataSet.
  * 
  * </pre>
  * 
@@ -529,91 +531,79 @@ public class ArchiveConfigManager {
      */
     private List<File> getDisplayFiles(DisplayData displayData, long startTime,
             long endTime) {
+        List<File> fileList = new LinkedList<File>();
         ArchiveConfig archiveConfig = displayData.archiveConfig;
-        CategoryConfig categoryConfig = displayData.categoryConfig;
 
-        String[] indexValues = categoryConfig.getDateGroupIndices().split(
+        for (CategoryDataSet dataSet : displayData.dataSets) {
-                "\\s*,\\s*");
-        int yearIndex = Integer.parseInt(indexValues[0]);
-        int monthIndex = Integer.parseInt(indexValues[1]);
-        int dayIndex = Integer.parseInt(indexValues[2]);
-        int hourIndex = Integer.parseInt(indexValues[3]);
 
-        String filePatternStr = categoryConfig.getFilePattern();
+            int[] timeIndices = dataSet.getTimeIndices();
 
-        boolean dirOnly = (filePatternStr == null)
+            String filePatternStr = dataSet.getFilePattern();
-                || ".*".equals(filePatternStr);
 
-        List<File> dirs = displayData.dirs;
+            boolean dirOnly = dataSet.isDirOnly();
 
-        int beginIndex = archiveConfig.getRootDir().length();
+            List<File> dirs = displayData.dirsMap.get(dataSet);
 
-        Calendar fileCal = TimeUtil.newCalendar();
+            int beginIndex = archiveConfig.getRootDir().length();
-        fileCal.setTimeZone(TimeZone.getTimeZone("UTC"));
 
-        List<File> fileList = new ArrayList<File>();
+            Calendar fileCal = TimeUtil.newCalendar();
+            fileCal.setTimeZone(TimeZone.getTimeZone("UTC"));
 
-        if (dirOnly) {
+            if (dirOnly) {
-            for (String dirPattern : categoryConfig.getDirPatternList()) {
+                for (String dirPattern : dataSet.getDirPatterns()) {
-                Pattern pattern = Pattern.compile(dirPattern);
+                    Pattern pattern = dataSet.getPattern(dirPattern);
 
-                for (File dir : dirs) {
+                    for (File dir : dirs) {
-                    String path = dir.getAbsolutePath().substring(beginIndex);
+                        String path = dir.getAbsolutePath().substring(
-                    Matcher matcher = pattern.matcher(path);
-                    if (matcher.matches()) {
-                        int year = Integer.parseInt(matcher.group(yearIndex));
-                        // Adjust month value to Calendar's 0 - 11
-                        int month = Integer.parseInt(matcher.group(monthIndex)) - 1;
-                        int day = Integer.parseInt(matcher.group(dayIndex));
-                        int hour = Integer.parseInt(matcher.group(hourIndex));
-                        fileCal.set(year, month, day, hour, 0, 0);
-                        long fileTime = fileCal.getTimeInMillis();
-                        if ((startTime <= fileTime) && (fileTime < endTime)) {
-                            fileList.add(dir);
-                        }
-                    }
-                }
-            }
-        } else {
-            for (String dirPattern : categoryConfig.getDirPatternList()) {
-                Pattern pattern = Pattern.compile(dirPattern + File.separator
-                        + categoryConfig.getFilePattern());
-                final Pattern filePattern = Pattern.compile("^"
-                        + filePatternStr + "$");
-                for (File dir : dirs) {
-                    List<File> fList = FileUtil.listDirFiles(dir,
-                            new FileFilter() {
-
-                                @Override
-                                public boolean accept(File pathname) {
-                                    return filePattern.matcher(
-                                            pathname.getName()).matches();
-                                }
-                            }, false);
-                    for (File file : fList) {
-                        String path = file.getAbsolutePath().substring(
                                 beginIndex);
                         Matcher matcher = pattern.matcher(path);
                         if (matcher.matches()) {
-                            int year = Integer.parseInt(matcher
+                            Long fileTime = dataSet.getMatchTimeInMilliseconds(
-                                    .group(yearIndex));
+                                    timeIndices, matcher);
-                            // Adjust month value to Calendar's 0 - 11
+                            if (fileTime == null) {
-                            int month = Integer.parseInt(matcher
+                                fileTime = dir.lastModified();
-                                    .group(monthIndex)) - 1;
+                            }
-                            int day = Integer.parseInt(matcher.group(dayIndex));
-                            int hour = Integer.parseInt(matcher
-                                    .group(hourIndex));
-                            fileCal.set(year, month, day, hour, 0, 0);
-                            long fileTime = fileCal.getTimeInMillis();
                             if ((startTime <= fileTime) && (fileTime < endTime)) {
-                                fileList.add(file);
+                                fileList.add(dir);
+                            }
+                        }
+                    }
+                }
+            } else {
+                for (String dirPattern : dataSet.getDirPatterns()) {
+                    Pattern pattern = dataSet.getPattern(dirPattern);
+                    final Pattern filePattern = Pattern.compile("^"
+                            + filePatternStr + "$");
+                    for (File dir : dirs) {
+                        List<File> fList = FileUtil.listDirFiles(dir,
+                                new FileFilter() {
+
+                                    @Override
+                                    public boolean accept(File pathname) {
+                                        return filePattern.matcher(
+                                                pathname.getName()).matches();
+                                    }
+                                }, false);
+                        for (File file : fList) {
+                            String path = file.getAbsolutePath().substring(
+                                    beginIndex);
+                            Matcher matcher = pattern.matcher(path);
+                            if (matcher.matches()) {
+                                Long timestamp = dataSet
+                                        .getMatchTimeInMilliseconds(
+                                                timeIndices, matcher);
+                                long fileTime = timestamp == null ? file
+                                        .lastModified() : timestamp.longValue();
+                                if ((startTime <= fileTime)
+                                        && (fileTime < endTime)) {
+                                    fileList.add(file);
+                                }
                             }
                         }
                     }
                 }
             }
         }
-
         return fileList;
     }
 
@@ -625,15 +615,13 @@ public class ArchiveConfigManager {
      * @param categoryConfig
      * @return dirs
      */
-    private List<File> getDirs(ArchiveConfig archiveConfig,
+    private List<File> getDirs(File rootFile, CategoryDataSet dataSet) {
-            CategoryConfig categoryConfig) {
         List<File> resultDirs = new ArrayList<File>();
-        File rootFile = new File(archiveConfig.getRootDir());
         List<File> dirs = new ArrayList<File>();
         List<File> tmpDirs = new ArrayList<File>();
         List<File> swpDirs = null;
 
-        for (String dirPattern : categoryConfig.getDirPatternList()) {
+        for (String dirPattern : dataSet.getDirPatterns()) {
             String[] subExpr = dirPattern.split(File.separator);
             dirs.clear();
             dirs.add(rootFile);
@@ -679,50 +667,59 @@ public class ArchiveConfigManager {
         Map<String, List<File>> displayMap = new HashMap<String, List<File>>();
 
         ArchiveConfig archiveConfig = archiveMap.get(archiveName);
+        String rootDirName = archiveConfig.getRootDir();
         CategoryConfig categoryConfig = findCategory(archiveConfig,
                 categoryName);
-        List<String> dirPatternList = categoryConfig.getDirPatternList();
+        File rootFile = new File(rootDirName);
+        TreeMap<String, DisplayData> displays = new TreeMap<String, DisplayData>();
+        for (CategoryDataSet dataSet : categoryConfig.getDataSetList()) {
+            List<String> dataSetDirPatterns = dataSet.getDirPatterns();
 
-        // index for making directory paths' relative to the root path.
+            List<File> dirs = getDirs(rootFile, dataSet);
-        List<File> dirs = getDirs(archiveConfig, categoryConfig);
 
-        File rootFile = new File(archiveMap.get(archiveName).getRootDir());
+            int beginIndex = rootFile.getAbsolutePath().length() + 1;
-        int beginIndex = rootFile.getAbsolutePath().length() + 1;
+            List<Pattern> patterns = new ArrayList<Pattern>(
-        List<Pattern> patterns = new ArrayList<Pattern>(dirPatternList.size());
+                    dataSetDirPatterns.size());
 
-        for (String dirPattern : dirPatternList) {
+            for (String dirPattern : dataSetDirPatterns) {
-            Pattern pattern = Pattern.compile("^" + dirPattern + "$");
+                Pattern pattern = Pattern.compile("^" + dirPattern + "$");
-            patterns.add(pattern);
+                patterns.add(pattern);
-        }
+            }
 
-        TreeSet<String> displays = new TreeSet<String>(
+            MessageFormat msgfmt = new MessageFormat(dataSet.getDisplayLabel());
-                String.CASE_INSENSITIVE_ORDER);
+            StringBuffer sb = new StringBuffer();
+            FieldPosition pos0 = new FieldPosition(0);
 
-        MessageFormat msgfmt = new MessageFormat(categoryConfig.getDisplay());
+            for (File dir : dirs) {
-        StringBuffer sb = new StringBuffer();
+                String path = dir.getAbsolutePath().substring(beginIndex);
-        FieldPosition pos0 = new FieldPosition(0);
+                for (Pattern pattern : patterns) {
-
+                    Matcher matcher = pattern.matcher(path);
-        for (File dir : dirs) {
+                    if (matcher.matches()) {
-            String path = dir.getAbsolutePath().substring(beginIndex);
+                        sb.setLength(0);
-            for (Pattern pattern : patterns) {
+                        String[] args = new String[matcher.groupCount() + 1];
-                Matcher matcher = pattern.matcher(path);
+                        args[0] = matcher.group();
-                if (matcher.matches()) {
+                        for (int i = 1; i < args.length; ++i) {
-                    sb.setLength(0);
+                            args[i] = matcher.group(i);
-                    String[] args = new String[matcher.groupCount() + 1];
+                        }
-                    args[0] = matcher.group();
+                        String displayLabel = msgfmt.format(args, sb, pos0)
-                    for (int i = 1; i < args.length; ++i) {
+                                .toString();
-                        args[i] = matcher.group(i);
+                        List<File> displayDirs = displayMap.get(displayLabel);
+                        if (displayDirs == null) {
+                            displayDirs = new ArrayList<File>();
+                            displayMap.put(displayLabel, displayDirs);
+                        }
+                        displayDirs.add(dir);
+                        DisplayData displayData = displays.get(displayLabel);
+                        if (displayData == null) {
+                            displayData = new DisplayData(archiveConfig,
+                                    categoryConfig, dataSet, displayLabel);
+                            displays.put(displayLabel, displayData);
+                        } else {
+                            displayData.dataSets.add(dataSet);
+                        }
+                        displayData.dirsMap.put(dataSet, displayDirs);
+                        break;
                     }
-                    String displayLabel = msgfmt.format(args, sb, pos0)
-                            .toString();
-                    List<File> displayDirs = displayMap.get(displayLabel);
-                    if (displayDirs == null) {
-                        displayDirs = new ArrayList<File>();
-                        displayMap.put(displayLabel, displayDirs);
-                    }
-                    displayDirs.add(dir);
-                    displays.add(displayLabel);
-                    break;
                 }
             }
         }
@@ -730,15 +727,7 @@ public class ArchiveConfigManager {
         List<DisplayData> displayDataList = new ArrayList<DisplayData>(
                 displays.size());
 
-        for (String displayLabel : displays) {
+        displayDataList.addAll(displays.values());
-            DisplayData displayData = new DisplayData(archiveConfig,
-                    categoryConfig, displayLabel, displayMap.get(displayLabel));
-            if (setSelect) {
-                displayData.setSelected(categoryConfig
-                        .getSelectedDisplayNames().contains(displayLabel));
-            }
-            displayDataList.add(displayData);
-        }
 
         return displayDataList;
     }

edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java

@@ -39,10 +39,13 @@ import javax.xml.bind.annotation.XmlRootElement;
  *   <name>redbook</name>
  *   <!-- When 0 default to the parent archive's retentionHours -->
  *   <retentionHours>0</retentionHours>
- *   <dirPattern>hdf5/(redbook)</dirPattern>
+ *   <dataSet>
- *   <displayLabel>{1}</displayLabel>
+ *      <dirPattern>hdf5/(redbook)</dirPattern>
- *   <filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+ *      <displayLabel>{1}</displayLabel>
- *   <dateGroupIndices>2,3,4,5</dateGroupIndices>
+ *      <filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+ *      <timeType>Date</timeType>
+ *      <dateGroupIndices>2,3,4,5</dateGroupIndices>
+ *   </dataSet>
  * </category>
  * </pre>
  * 
@@ -52,9 +55,12 @@ import javax.xml.bind.annotation.XmlRootElement;
  * &lt;category>
  *   &lt;name>Model grib&lt;/name>
  *   &lt;retentionHours>0&lt;/retentionHours>
- *   &lt;dirPattern>grib/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)&lt;/dirPattern>
+ *   &lt;dataSet>
- *   &lt;displayLabel>{5}&lt;/displayLabel>
+ *      &lt;dirPattern>grib/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)&lt;/dirPattern>
- *   &lt;dateGroupIndices>1,2,3,4&lt;/dateGroupIndices>
+ *      &lt;displayLabel>{5}&lt;/displayLabel>
+ *      &lt;timeType>Date&lt;/timeType>
+ *      &lt;dateGroupIndices>1,2,3,4&lt;/dateGroupIndices>
+ *   &lt/dataSet>
  * &lt;/category>
  * </pre>
  * 
@@ -65,6 +71,7 @@ import javax.xml.bind.annotation.XmlRootElement;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * May 1, 2013  1966       rferrel     Initial creation
+ * Aug 03, 2013 2224       rferrel     Changes to include DataSet.
  * 
  * </pre>
  * 
@@ -88,63 +95,8 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
     @XmlElement(name = "extRetentionHours")
     private int retentionHours;
 
-    /**
+    @XmlElement(name = "dataSet")
-     * A regex pattern to find directories controlled by the category. These
+    private List<CategoryDataSet> dataSetList;
-     * directories should be relative to the parent archive's directory. For
-     * example:
-     * 
-     * <pre>
-     * &lt;dirPattern>grib2/\d{8}/\d{2}/(.*)/&lt;/dirPattern>
-     * </pre>
-     */
-    @XmlElement(name = "dirPattern")
-    private List<String> dirPatternList;
-
-    /**
-     * Use to display the information found by the dirPattern. Any groups in the
-     * dirPatern may be displayed. For example:
-     * 
-     * <pre>
-     * &lt;dirName>(grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)&lt;/dirName>
-     * &lt;displayLabel>{1} - {6}&lt;/displayLabel>
-     * </pre>
-     * 
-     * The {1} will be replaced by the first group (grib2) in the regex
-     * expression in dirName. The {6} is the sixth group (.*). {0} is the whole
-     * expression match.
-     */
-    @XmlElement(name = "displayLabel")
-    private String display;
-
-    /**
-     * A comma separated list of 4 numbers representing the group indices
-     * specifying the location of the numeric date time stamp. The indices must
-     * be in the order of year, month, day and hour. The group numbering starts
-     * with the first group in the dirPattern and continues with any grouping in
-     * the filePattern.
-     * 
-     * <pre>
-     *   &lt;dirPattern>hdf5/(redbook)&lt;/dirPattern>
-     *   &lt;displayLabel>{1}&lt;/displayLabel>
-     *   &lt;filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*&lt;/filePattern>
-     *   &lt;dateGroupIndices>2,3,4,5&lt;/dateGroupIndices>
-     * </pre>
-     */
-    @XmlElement(name = "dateGroupIndices")
-    private String dateGroupIndices;
-
-    /**
-     * A saveDir directory may contain files with data for several days. This
-     * allows any of the year, month, day and hour to be part of a file name.
-     * Default is all files in the directory. For example:
-     * 
-     * <pre>
-     * &lt;saveDir>hd5/redbook/(^/]*&#47/)&lt/saveDir>
-     * &lt;saveFile>redbook-${YYYY}-${MM}-${DD}-${HH}\..*&lt;saveFiles>
-     * </pre>
-     */
-    @XmlElement(name = "filePattern")
-    private String filePattern;
 
     @XmlElement(name = "selectedDisplayName")
     private final Collection<String> selectedDisplayNames = new TreeSet<String>();
@@ -188,76 +140,12 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
         this.retentionHours = retentionHours;
     }
 
-    /**
+    public List<CategoryDataSet> getDataSetList() {
-     * Obtain the list of directory patterns.
+        return new ArrayList<CategoryDataSet>(dataSetList);
-     * 
-     * @return dirPatternList
-     */
-    public List<String> getDirPatternList() {
-        return new ArrayList<String>(dirPatternList);
     }
 
-    /**
+    public void setDataSetList(List<CategoryDataSet> dataSetList) {
-     * Set the directory pattern list; must not be null.
+        this.dataSetList = dataSetList;
-     * 
-     * @param dirPatternList
-     */
-    public void setDirPatternList(List<String> dirPatternList) {
-        this.dirPatternList = dirPatternList;
-    }
-
-    /**
-     * Get the display label pattern.
-     * 
-     * @return display
-     */
-    public String getDisplay() {
-        return display == null ? "" : display;
-    }
-
-    /**
-     * Set the display label pattern.
-     * 
-     * @param display
-     */
-    public void setDisplay(String display) {
-        this.display = display;
-    }
-
-    /**
-     * Get the save directory pattern..
-     * 
-     * @return dateGroups
-     */
-    public String getDateGroupIndices() {
-        return dateGroupIndices;
-    }
-
-    /**
-     * Set the save directory pattern; must not be null.
-     * 
-     * @param saveDir
-     */
-    public void setDateGroupIndices(String dateGroupIndices) {
-        this.dateGroupIndices = dateGroupIndices;
-    }
-
-    /**
-     * Get the save files pattern.
-     * 
-     * @return saveFiles
-     */
-    public String getFilePattern() {
-        return filePattern;
-    }
-
-    /**
-     * Set the save files pattern; may be null.
-     * 
-     * @param saveFiles
-     */
-    public void setFilePattern(String filePattern) {
-        this.filePattern = filePattern;
     }
 
     public Collection<String> getSelectedDisplayNames() {
@@ -285,9 +173,9 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
      */
     @Override
     public int compareTo(CategoryConfig o) {
-        return getDisplay().compareToIgnoreCase(o.getDisplay());
+        return getName().compareToIgnoreCase(o.getName());
     }
-    
+
     /*
      * (non-Javadoc)
      * 
@@ -298,13 +186,10 @@ public class CategoryConfig implements Comparable<CategoryConfig> {
         StringBuilder sb = new StringBuilder();
         sb.append("Category [ name: ").append(getName());
         sb.append(", retentionHours: ").append(getRetentionHours());
-        sb.append(", dirPatternList[ ");
+        sb.append(", dataSetList[ ");
-        for (String dirPattern : getDirPatternList()) {
+        for (CategoryDataSet dataSet : getDataSetList()) {
-            sb.append(" \"").append(dirPattern).append("\",");
+            sb.append(dataSet).append(", ");
         }
-        sb.append("], filePattern: ").append(getFilePattern());
-        sb.append(", displayLabel: ").append(getDisplay());
-        sb.append(", dateGroupIndices: ").append(getDateGroupIndices());
         sb.append(", selectedDisplayNames: ");
         if (selectedDisplayNames == null) {
             sb.append("null");

edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryDataSet.java

@@ -0,0 +1,280 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.common.archive.config;
+
+import java.io.File;
+import java.util.Calendar;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.raytheon.uf.common.time.util.TimeUtil;
+
+/**
+ * A grouping of data set information used in a category.
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 6, 2013  #2224      rferrel     Initial creation
+ * 
+ * </pre>
+ * 
+ * @author rferrel
+ * @version 1.0
+ */
+@XmlAccessorType(XmlAccessType.NONE)
+@XmlRootElement(name = "dataSet")
+public class CategoryDataSet {
+    public static final int YEAR_INDEX = 0;
+
+    public static final int MONTH_INDEX = 1;
+
+    public static final int DAY_INDEX = 2;
+
+    public static final int HOUR_INDEX = 3;
+
+    public static final int TIMESTAMP_INDEX = 0;
+
+    /**
+     * Types of times and the number of indices for getting the time stamp from
+     * patterns.
+     */
+    public static enum TimeType {
+        Date(4), EpochSec(1), EpochMS(1), File(0);
+
+        private final int numIndices;
+
+        private TimeType(int numIndices) {
+            this.numIndices = numIndices;
+        }
+
+        public int getNumGroupIndices() {
+            return numIndices;
+        }
+    }
+
+    /**
+     * List of directory patterns.
+     */
+    @XmlElement(name = "dirPattern")
+    private List<String> dirPatterns;
+
+    /**
+     * Optional file pattern.
+     */
+    @XmlElement(name = "filePattern")
+    private String filePattern;
+
+    @XmlElement(name = "timeType")
+    private TimeType timeType;
+
+    /**
+     * The display label.
+     */
+    @XmlElement(name = "displayLabel")
+    private String displayLabel;
+
+    private int[] timeIndices = null;
+
+    /**
+     * The index with group number for getting desired parts of the TimeType.
+     */
+    @XmlElement(name = "dateGroupIndices")
+    private String dateGroupIndices = "";
+
+    public List<String> getDirPatterns() {
+        return dirPatterns;
+    }
+
+    public void setDirPatterns(List<String> dirPatterns) {
+        this.dirPatterns = dirPatterns;
+    }
+
+    public String getFilePattern() {
+        return filePattern;
+    }
+
+    public void setFilePattern(String filePattern) {
+        this.filePattern = filePattern;
+    }
+
+    public TimeType getTimeType() {
+        return timeType;
+    }
+
+    public void setTimeType(TimeType timeType) {
+        this.timeType = timeType;
+    }
+
+    public String getDisplayLabel() {
+        return displayLabel;
+    }
+
+    public void setDisplayLabel(String displayLabel) {
+        this.displayLabel = displayLabel;
+    }
+
+    public String getDateGroupIndices() {
+        return dateGroupIndices;
+    }
+
+    public void setDateGroupIndices(String dateGroupIndices) {
+        this.dateGroupIndices = dateGroupIndices;
+        this.timeIndices = null;
+    }
+
+    /**
+     * Get the array of time indices based on time type and date group indices.
+     * 
+     * @return timeIndices
+     */
+    public int[] getTimeIndices() {
+        if (timeIndices == null) {
+            timeIndices = new int[timeType.getNumGroupIndices()];
+            if (timeIndices.length > 0) {
+                String[] indexValues = getDateGroupIndices().split("\\s*,\\s*");
+                for (int index = 0; index < timeIndices.length; ++index) {
+                    timeIndices[index] = Integer.parseInt(indexValues[index]);
+                }
+            }
+        }
+        return timeIndices;
+    }
+
+    /**
+     * Get the Pattern for dirPattern.
+     * 
+     * @param dirPattern
+     * 
+     * @return pattern or null if dirPattern not in the list of directory
+     *         patterns.
+     */
+    public Pattern getPattern(String dirPattern) {
+        Pattern pattern = null;
+        if (dirPatterns.contains(dirPattern)) {
+            if (isDirOnly()) {
+                pattern = Pattern.compile(dirPattern);
+            } else {
+                pattern = Pattern.compile(dirPattern + File.separator
+                        + getFilePattern());
+            }
+        }
+        return pattern;
+    }
+
+    /**
+     * 
+     * @return true when only the dirPatterns should be used.
+     */
+    public boolean isDirOnly() {
+        return filePattern == null || filePattern.equals(".*");
+    }
+
+    /**
+     * Get time stamp for file based on timetype. Assumes file path matches
+     * 
+     * @param timeIndices
+     * @param matcher
+     * @param file
+     * @return
+     */
+    public Long getMatchTimeInMilliseconds(int[] timeIndices, Matcher matcher) {
+        return CategoryDataSet.getMatchTimeInMilliseconds(timeType,
+                timeIndices, matcher);
+    }
+
+    /**
+     * Get file time based on time type.
+     * 
+     * @param timeType
+     * @param timeIndices
+     * @param matcher
+     * @param file
+     * @return fileTime
+     */
+    public static Long getMatchTimeInMilliseconds(
+            CategoryDataSet.TimeType timeType, int[] timeIndices,
+            Matcher matcher) {
+        Long fileTime = null;
+        switch (timeType) {
+        case Date:
+            Calendar fileCal = TimeUtil.newGmtCalendar();
+            int year = Integer.parseInt(matcher
+                    .group(timeIndices[CategoryDataSet.YEAR_INDEX]));
+            // Adjust month value to Calendar's 0 - 11
+            int month = Integer.parseInt(matcher
+                    .group(timeIndices[CategoryDataSet.MONTH_INDEX])) - 1;
+            int day = Integer.parseInt(matcher
+                    .group(timeIndices[CategoryDataSet.DAY_INDEX]));
+            int hour = Integer.parseInt(matcher
+                    .group(timeIndices[CategoryDataSet.HOUR_INDEX]));
+            fileCal.set(year, month, day, hour, 0, 0);
+            fileTime = fileCal.getTimeInMillis();
+            break;
+        case EpochMS:
+            fileTime = Long.parseLong(matcher
+                    .group(timeIndices[CategoryDataSet.TIMESTAMP_INDEX]));
+            break;
+        case EpochSec:
+            fileTime = Long.parseLong(matcher
+                    .group(timeIndices[CategoryDataSet.TIMESTAMP_INDEX]));
+            fileTime *= TimeUtil.MILLIS_PER_SECOND;
+            break;
+        case File:
+            fileTime = null;
+            break;
+        default:
+            fileTime = null;
+            break;
+        }
+        return fileTime;
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder("DataSet[ ");
+        sb.append("TimeType: ").append(getTimeType());
+        sb.append("dateGroupIndices: ").append(getDateGroupIndices());
+        sb.append(", isDirOnly: ").append(isDirOnly());
+        sb.append(", displayLabel: ").append(getDisplayLabel());
+        sb.append(", dirPatterns[ ");
+        for (String dirPattern : getDirPatterns()) {
+            sb.append(dirPattern).append(", ");
+        }
+        sb.append("], filePattern: ").append(
+                filePattern == null ? "null" : filePattern);
+        sb.append("]");
+        return sb.toString();
+    }
+}

edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryFileDateHelper.java

@@ -40,6 +40,7 @@ import com.raytheon.uf.common.time.util.TimeUtil;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Jun 21, 2013 1965       bgonzale    Initial creation
+ * Aug 03, 2013 2224       rferrel     Changes for new configuration files.
  * 
  * </pre>
  * 
@@ -56,13 +57,11 @@ public class CategoryFileDateHelper implements IFileDateHelper {
 
         private final Pattern categoryTopLevelDirPattern;
 
-        private final int yearIndex;
+        private final CategoryDataSet.TimeType timeType;
 
-        private final int monthIndex;
+        private final boolean isDirOnly;
 
-        private final int dayIndex;
+        private final int[] timeIndices;
-
-        private final int hourIndex;
 
         /**
          * Initialization constructor.
@@ -76,13 +75,13 @@ public class CategoryFileDateHelper implements IFileDateHelper {
          */
         public CategoryDateInfo(Pattern datePattern,
                 Pattern categoryTopLevelDirPattern,
-                int yearIndex, int monthIndex, int dayIndex, int hourIndex) {
+                CategoryDataSet.TimeType timeType, boolean isDirOnly,
+                int[] timeIndices) {
             this.datePattern = datePattern;
             this.categoryTopLevelDirPattern = categoryTopLevelDirPattern;
-            this.yearIndex = yearIndex;
+            this.timeType = timeType;
-            this.monthIndex = monthIndex;
+            this.isDirOnly = isDirOnly;
-            this.dayIndex = dayIndex;
+            this.timeIndices = timeIndices;
-            this.hourIndex = hourIndex;
         }
 
     }
@@ -91,8 +90,6 @@ public class CategoryFileDateHelper implements IFileDateHelper {
 
     private final String rootDir;
 
-    private final boolean isDirOnly;
-
     /**
      * Initialization constructor.
      * 
@@ -101,38 +98,37 @@ public class CategoryFileDateHelper implements IFileDateHelper {
      *            categoryTopLevelDirPattern
      */
     public CategoryFileDateHelper(CategoryConfig config, String rootDir) {
-        List<String> categoryDirPatternList = config.getDirPatternList();
-        this.dateInfoList = new ArrayList<CategoryFileDateHelper.CategoryDateInfo>(
-                categoryDirPatternList.size());
-
-        String filePatternStr = config.getFilePattern();
         this.rootDir = rootDir;
-        this.isDirOnly = (filePatternStr == null)
+        List<CategoryDataSet> categoryDataSetList = config.getDataSetList();
-                || ".*".equals(filePatternStr);
+        int size = 0;
+        for (CategoryDataSet dataSet : categoryDataSetList) {
+            size += dataSet.getDirPatterns().size();
+        }
 
-        for (String patternString : categoryDirPatternList) {
+        this.dateInfoList = new ArrayList<CategoryFileDateHelper.CategoryDateInfo>(
-            Pattern datePattern = null;
+                size);
-            if (isDirOnly) {
+
-                datePattern = Pattern.compile(patternString);
+        boolean isDirOnly;
-            } else {
+        CategoryDataSet.TimeType timeType;
-                datePattern = Pattern.compile(patternString + File.separator
+        for (CategoryDataSet dataSet : categoryDataSetList) {
-                        + config.getFilePattern());
+            isDirOnly = dataSet.isDirOnly();
+            timeType = dataSet.getTimeType();
+
+            for (String patternString : dataSet.getDirPatterns()) {
+                Pattern datePattern = dataSet.getPattern(patternString);
+                int dirSeparatorIndex = patternString
+                        .indexOf(File.separatorChar);
+                patternString = dirSeparatorIndex > patternString.length()
+                        || dirSeparatorIndex < 0 ? patternString
+                        : patternString.substring(0, dirSeparatorIndex);
+                Pattern categoryTopLevelDirPattern = Pattern
+                        .compile(patternString);
+                int[] timeIndices = dataSet.getTimeIndices();
+
+                dateInfoList.add(new CategoryDateInfo(datePattern,
+                        categoryTopLevelDirPattern, timeType, isDirOnly,
+                        timeIndices));
             }
-            int dirSeparatorIndex = patternString.indexOf(File.separatorChar);
-            patternString = dirSeparatorIndex > patternString.length()
-                    || dirSeparatorIndex < 0 ? patternString : patternString
-                    .substring(0, dirSeparatorIndex);
-            Pattern categoryTopLevelDirPattern = Pattern.compile(patternString);
-            String[] indexValues = config.getDateGroupIndices().split(
-                    "\\s*,\\s*");
-            int yearIndex = Integer.parseInt(indexValues[0]);
-            int monthIndex = Integer.parseInt(indexValues[1]);
-            int dayIndex = Integer.parseInt(indexValues[2]);
-            int hourIndex = Integer.parseInt(indexValues[3]);
-
-            dateInfoList.add(new CategoryDateInfo(datePattern,
-                    categoryTopLevelDirPattern, yearIndex, monthIndex,
-                    dayIndex, hourIndex));
         }
     }
 
@@ -145,36 +141,39 @@ public class CategoryFileDateHelper implements IFileDateHelper {
      */
     @Override
     public Calendar getFileDate(String filenamePath) {
-        String pathForPatternCheck = filenamePath.substring(rootDir.length());
+        String pathForFilePatternCheck = filenamePath.substring(rootDir
-        pathForPatternCheck = isDirOnly ? FilenameUtils
+                .length());
-                .getFullPathNoEndSeparator(pathForPatternCheck)
+        String pathForDirPatternCheck = FilenameUtils
-                : pathForPatternCheck;
+                .getFullPathNoEndSeparator(pathForFilePatternCheck);
         Calendar result = null;
+        Long timestamp = null;
 
         for (CategoryDateInfo dateInfo : dateInfoList) {
-            Matcher matcher = dateInfo.datePattern.matcher(pathForPatternCheck);
+            Matcher matcher = null;
+            if (dateInfo.isDirOnly) {
+                matcher = dateInfo.datePattern.matcher(pathForDirPatternCheck);
+            } else {
+                matcher = dateInfo.datePattern.matcher(pathForFilePatternCheck);
+            }
 
             if (matcher.matches()) {
-                int year = Integer.parseInt(matcher.group(dateInfo.yearIndex));
+                timestamp = CategoryDataSet.getMatchTimeInMilliseconds(
-                // Adjust month value to Calendar's 0 - 11
+                        dateInfo.timeType, dateInfo.timeIndices, matcher);
-                int month = Integer
-                        .parseInt(matcher.group(dateInfo.monthIndex)) - 1;
-                int day = Integer.parseInt(matcher.group(dateInfo.dayIndex));
-                int hour = Integer.parseInt(matcher.group(dateInfo.hourIndex));
-
-                result = TimeUtil.newGmtCalendar();
-                result.set(year, month, day, hour, 0, 0);
                 break;
             }
         }
-        if (result == null) {
+
+        if (timestamp == null) {
             // no matching pattern, use file last modified date
             File file = new File(filenamePath);
-            long lastModifiedMillis = file.lastModified();
+            timestamp = file.lastModified();
-
-            result = TimeUtil.newGmtCalendar();
-            result.setTimeInMillis(lastModifiedMillis);
         }
+
+        // TODO future speed improvement refactor IFileDateHelper to have a
+        // method that returns a long instead of Calendar. That will prevent
+        // converting Calendar to long then back to a Calendar.
+        result = TimeUtil.newGmtCalendar();
+        result.setTimeInMillis(timestamp);
         return result;
     }
 

edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/DisplayData.java

@@ -1,8 +1,11 @@
 package com.raytheon.uf.common.archive.config;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import com.raytheon.uf.common.util.SizeUtil;
 
@@ -18,6 +21,7 @@ import com.raytheon.uf.common.util.SizeUtil;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Jun 7, 2013  1966       rferrel     Initial creation
+ * Aug 02, 2013 2224       rferrel     Changes to include DataSet in configuration.
  * Aug 06, 2013 2222       rferrel     Changes to display all selected data.
  * 
  * </pre>
@@ -71,22 +75,26 @@ public class DisplayData implements Comparable<DisplayData> {
     /** The data's category configuration. */
     protected final CategoryConfig categoryConfig;
 
+    protected final List<CategoryDataSet> dataSets = new ArrayList<CategoryDataSet>();
+
     /** The display label for this data. */
     protected final String displayLabel;
 
     /**
-     * List of directories for the display label matching the category's
+     * Mappings of a list of directories for the display label matching the data
-     * directory pattern and found under the archive's root directory.
+     * set's directory patterns and found under the archive's root directory.
      */
-    protected final List<File> dirs;
+    protected final Map<CategoryDataSet, List<File>> dirsMap = new HashMap<CategoryDataSet, List<File>>();
 
     /**
-     * For use by GUI to indicate. Use to indicate selected for retention or for
+     * For use by GUI to indicate display label's row is selected.
-     * placing in a case.
      */
     private boolean selected = false;
 
-    /** For use by GUI for indicating the size of the directories' contents. */
+    /**
+     * For use by GUI for indicating the size of the display label's row
+     * contents.
+     */
     private long size = UNKNOWN_SIZE;
 
     /**
@@ -94,15 +102,14 @@ public class DisplayData implements Comparable<DisplayData> {
      * 
      * @param archiveConfig
      * @param categoryConfig
+     * @param dataSet
      * @param displayLabel
-     * @param dirs
      */
     public DisplayData(ArchiveConfig archiveConfig,
-            CategoryConfig categoryConfig, String displayLabel, List<File> dirs) {
+            CategoryConfig categoryConfig, CategoryDataSet dataSet, String displayLabel) {
         this.archiveConfig = archiveConfig;
         this.categoryConfig = categoryConfig;
         this.displayLabel = displayLabel;
-        this.dirs = dirs;
     }
 
     /**

edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archiver/purger/PROCESSED_DATA.xml

@@ -24,43 +24,60 @@
  * Date         Ticket#    Engineer    Description
  * ============ ========== =========== ==========================
  * Jun 20, 2013 1966       rferrel     Initial creation
+ * Aug 05, 2012 2224       rferrel     Changes to add dataSet tags.
  * 
  * @author rferrel
  * @version 1.0
 -->
     <!-- 
-    The <archive> contains five types of tags:
+    The <archive> contains four types of tags:
         <name>           - Required tag. The id for the archive such as Raw or Processed.
                            Used in the GUIs to filter which archive to display.
         <rootDir>        - Required tag. The root directory on the edex server for the archive.
         <minRetentionHours> - Required tag. The default number of hours to retain data in the <rootDir>
         <category>       - Logical grouping of archive's sub-directories.
     
-    The <category> has six types of tags:
+    The <category> has four types of tags:
         <name>              - Required tag. The id for the category such as grib2 or redbook.
                               Used in the GUIs to filter what is the display in the table.
         <extRetentionHours>    - Optional tag. The  extended retentionHours for selected directories of this category.
                               Default is the archive's <minRetentionHours>.
-        <dirPattern>        - Required tag. A regex pattern for finding directories for this category.
+        <dataSet>           - Required to have a least one. Each one contains a set of tags explained below.
+        <selectedDisplayNames>  - A directory matching <dirPattern>. These are selected directories from the Retention GUI. The purger
+                                  will used the category's <extRetentionHours> instead of the Arhivie's <minRetentionHours>.
+                                  An optional tag; may have more then one. (NOTE these are set internally when a selection configuration
+                                  file is loaded. They should not appear in the configuration file.)
+     The <dataSet> contains ? types of tags:
+        <dirPattern>        - Required to have at least one. A regex pattern for finding directories for this category.
                               The pattern is relative to the archive's <rootDir>. Wildcard patterns do not cross directory
                               delimiter /. Thus to match 3 levels of directories you would need .*/.*/.* .
-                              See patterns and groups section. There may be more then one of these tags. The restriction is they
+                              See patterns and groups section. There may be more then one of these tags in a <dataSet>. The
-                              must all have the same number of groupings and be in the same order.
+                              restriction is they must all have the same number of groupings and be in the same order to match up
+                              with the <displayLabel>'s, and <dateGroupIndicies>'s values.
         <filePattern>       - Optional tag.  A pattern to find files in the directories that match the <dirPattern>.
                               Default is everything in the directories that match <dirPattern>.
                               See patterns and groups section.
         <displayLabel>      - Required tag. The label to display for directories that match <dirPattern>. Any group in the
                               <dirPattern> may be made part of the label by placing the group's index inside parenthesis, {1}.
-                              More then one directory may match the <dirPattern>. The archive GUIs may  collapse them into a
+                              More then one directory may match the <dirPattern>. The archive GUIs may collapse them into a
                               single table entry.
-        <dateGroupIndicies> - Required tag. A comma separated list of 4 numbers which are the index of the groups in
+        <timeType>          - Optional tag to determine what type of time stamp is being used to get files/directories for retention
-                              <dirPattern> and/or <filePattern> that contain in order the year, mouth, day and hour information.
+                              and case creation. The value dictates how many groupings in the <dirPattern>s and/or <filePattern> are
+                              used to get the time stamp for a file. The four values are:
+                              Date - (default) the time stamp is made up of four groups in the patterns: year, month, day and hour.
+                              EpochSec - The time stamp has one group in the patterns which is the epoch time in seconds.
+                              EpochMS - The time stamp has one group in the patterns which is the epoch time in milliseconds.
+                              File - No group is used to get the time stamp. Instead use the files date of last modification.
+        <dateGroupIndicies> - Required tag when <timeType> has any value but File. 
+                              Date - A comma separated list of 4 numbers which are in order the index for year, month, day and hour.
+                              EpochSec - A number which is the index for the epoch in seconds.
+                              EpochMS - A number which is the index for the epoch in milliseconds.
+                              File - Not needed since no group is used to get the time stamp.
+                              
                               This is used to determine what files/directories to retain or a range of directories/files to copy
                               for case creation. Note to get the group's index the <dirPattern> and <filePattern> are combined.
                               Thus if there are 5 groups in the <dirPattern> then the first group in the <filePattern> is index 6.
-        <selectedDisplayNames>  - A directory matching <dirPattern>. These are selected directories from the Retention GUI. The purger
+     
-                                  will used the category's <extRetentionHours> instead of the Arhivie's <minRetentionHours>.
-                                  An optional tag; may have more then one.
      Patterns and groups.
      The <dirPattern> and <filePattern> use Java regex expressions; similar to the ldm's pqact.conf file.
      For more details see http://docs.oracle.com/javase/tutorial/essential/regex/
@@ -77,7 +94,7 @@
      Finally the sixth group will match any sub-directory that in the hour directory.
      
      Thus the directory paths <rootPath>/grib2/20130527/18/GFS will generate the display string, grib2 - GFS, and from the grouping we
-     can find the year, 201; month, 05; day, 27 and hour, 18.
+     can find the year, 2013; month, 05; day, 27 and hour, 18.
      
      Example with <filePattern>:
         <dirPattern>hdf5/(redbook)</dirPattern>
@@ -89,11 +106,14 @@
      <category>
         <name>Observation</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>(acars|airep|airmet|taf)</dirPattern>
+        <dataSet>
-        <dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
+           <dirPattern>(acars|airep|airmet|taf)</dirPattern>
-        <displayLabel>{1}</displayLabel>
+           <dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
-        <dateGroupIndices>2,3,4,5</dateGroupIndices>
+           <displayLabel>{1}</displayLabel>
-        <filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+           <timeType>Date</timeType>
+           <dateGroupIndices>2,3,4,5</dateGroupIndices>
+           <filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+        </dataSet>
      </category>
      The first <dirPattern> looks for files matching the <filePattern> in the directories acars, airep, airmet or taf.
      The second <dirPattern> expects to find the files in subdirectories of bufrsigwx or sfcobs such as bufrsigwx/SWH.
@@ -110,79 +130,111 @@
     <rootDir>/awips2/edex/data/archive/</rootDir>
     <minRetentionHours>24</minRetentionHours>
      <category>
-        <name>DAT</name>
+        <name>Decision Assistance</name>
          <extRetentionHours>168</extRetentionHours>
-         <dirPattern>(cwat|fog|ffmp|preciprate|qpf|scan|vil)</dirPattern>
+         <dataSet>
-         <filePattern>.*(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
+            <dirPattern>(cwat|fog|ffmp|fssobs|preciprate|qpf|scan|vil)</dirPattern>
-         <displayLabel>{1}</displayLabel>
+            <filePattern>.*(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
-         <dateGroupIndices>2,3,4,5</dateGroupIndices>
+            <displayLabel>{1}</displayLabel>
+            <dateGroupIndices>2,3,4,5</dateGroupIndices>
+        </dataSet>
     </category>
     <category>
-        <name>gfe</name>
+        <name>GFE</name>
          <extRetentionHours>168</extRetentionHours>
-         <dirPattern>(gfe)/(.*)/.*</dirPattern>
+         <dataSet>
-         <dirPattern>(gfe)/(.*)</dirPattern>
+            <dirPattern>(gfe)/(.*)/.*</dirPattern>
-         <filePattern>.*_(\d{4})(\d{2})(\d{2})_(\d{2}).*</filePattern>
+            <dirPattern>(gfe)/(.*)</dirPattern>
-         <displayLabel>{2}</displayLabel>
+            <filePattern>.*_(\d{4})(\d{2})(\d{2})_(\d{2}).*</filePattern>
-         <dateGroupIndices>3,4,5,6</dateGroupIndices>
+            <displayLabel>{2}</displayLabel>
+            <dateGroupIndices>3,4,5,6</dateGroupIndices>
+         </dataSet>
+    </category>
+    <category>
+        <name>Local</name>
+        <extRetentionHours>168</extRetentionHours>
+        <dataset>
+            <dirPattern>(ldadhydro|ldadmesonet|ldadprofiler|ldad_manual|mesowest|qc)</dirPattern>
+            <filePattern>.*(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
+            <displayLabel>{1}</displayLabel>
+            <dateGroupIndices>2,3,4,5</dateGroupIndices>
+        </dataset>
     </category>
     <category>
         <name>Model</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>(grid)/(.*)/(.*)</dirPattern>
+        <dataSet>
-        <displayLabel>{2}</displayLabel>
+            <dirPattern>(grid)/(.*)/(.*)</dirPattern>
-        <dateGroupIndices>4,5,6,7</dateGroupIndices>
+            <displayLabel>{2}</displayLabel>
-        <filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})-.*</filePattern>
+            <dateGroupIndices>4,5,6,7</dateGroupIndices>
+            <filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})-.*</filePattern>
+        </dataSet>
+        <dataSet>
+            <dirPattern>(modelsounding)/(.*)</dirPattern>
+            <dirPattern>(bufrmos)(.*)</dirPattern>
+            <displayLabel>{1} - {2}</displayLabel>
+            <dateGroupIndices>3,4,5,6</dateGroupIndices>
+            <filePattern>.*(\d{4})-(\d{2})-(\d{2})[-_](\d{2}).*</filePattern>
+        </dataSet>
     </category>
     <category>
-        <name>Misc</name>
+        <name>Products</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>(binlightning|ccfp|cwa|idft|lsr|manual|svrwx|tcg|tcm|tcs|text|warning|wcp)</dirPattern>
+        <dataSet>
-        <dirPattern>(redbook)/.*</dirPattern>
+            <dirPattern>(airmet|atcf|aww|bufrncwf|ccfp|convsigmet|cwa|ffg|intlsigmet|nonconvsigmet|stormtrack|taf|tcg|tcm|tcs|text|vaa|warning|wcp)</dirPattern>
-        <displayLabel>{1}</displayLabel>
+            <dirPattern>(bufrsigwx|redbook)/.*</dirPattern>
-        <dateGroupIndices>2,3,4,5</dateGroupIndices>
+            <displayLabel>{1}</displayLabel>
-        <filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+            <dateGroupIndices>2,3,4,5</dateGroupIndices>
+            <filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+        </dataSet>
     </category>
     <category>
         <name>Observation</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>(acars|airep|airmet|bufrascat|bufrhdw|bufrmthdw|bufrncwf|bufrssmi|convsigmet|fssobs|intlsigmet|nonconvsigmet|obs|pirep|taf)</dirPattern>
+        <dataSet>
-        <dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
+            <dirPattern>(acars|airep|binlightning|bufrascat|bufrhdw|bufrmthdw|bufrssmi|idft|lsr|obs|pirep|recco|svrwx)</dirPattern>
-        <displayLabel>{1}</displayLabel>
+            <dirPattern>(sfcobs)/.*</dirPattern>
-        <dateGroupIndices>2,3,4,5</dateGroupIndices>
+            <displayLabel>{1}</displayLabel>
-        <filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+            <dateGroupIndices>2,3,4,5</dateGroupIndices>
+            <filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+        </dataSet>
     </category>
     <category>
         <name>Satellite</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>satellite/(.*)/(.*)</dirPattern>
+        <dataSet>
-        <filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
+            <dirPattern>satellite/(.*)/(.*)</dirPattern>
-        <dateGroupIndices>3,4,5,6</dateGroupIndices>
+            <filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
-        <displayLabel>{1}</displayLabel>
+            <dateGroupIndices>3,4,5,6</dateGroupIndices>
+            <displayLabel>{1}</displayLabel>
+        </dataSet>
+        <dataSet>
+            <!-- Guess for mcidas and viirs is based on old example. -->
+            <dirPattern>(mcidas|viirs)/.*/.*/.*/.*</dirPattern>
+            <filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2}).*</filePattern>
+            <dateGroupIndices>2,3,4,5</dateGroupIndices>
+            <displayLabel>{1}</displayLabel>
+        </dataSet>
     </category>
     <category>
-        <name>Sounding</name>
+        <name>Profiles</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>(acarssounding|bufrua|goessounding|poessounding|profiler|raobs)</dirPattern>
+        <dataSet>
-        <displayLabel>{1}</displayLabel>
+            <dirPattern>(acarssounding|bufrua|goessounding|poessounding|profiler)</dirPattern>
-        <dateGroupIndices>2,3,4,5</dateGroupIndices>
+            <displayLabel>{1}</displayLabel>
-        <filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+            <dateGroupIndices>2,3,4,5</dateGroupIndices>
-    </category>
+            <filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
-     <category>
+        </dataSet>
-        <name>ModelSounding</name>
-        <extRetentionHours>168</extRetentionHours>
-        <dirPattern>(modelsounding)/(.*)</dirPattern>
-        <dirPattern>(bufrmos)(.*)</dirPattern>
-        <displayLabel>{1} - {2}</displayLabel>
-        <dateGroupIndices>3,4,5,6</dateGroupIndices>
-        <filePattern>.*(\d{4})-(\d{2})-(\d{2})[-_](\d{2}).*</filePattern>
     </category>
     <category>
+    <!-- Find mosaic dataset and add here. -->
         <name>radar</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>radar/(.*)/(.*)</dirPattern>
+        <dataSet>
-        <displayLabel>{1}</displayLabel>
+            <dirPattern>radar/(.*)/(.*)</dirPattern>
-        <dateGroupIndices>3,4,5,6</dateGroupIndices>
+            <displayLabel>{1}</displayLabel>
-        <filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+            <dateGroupIndices>3,4,5,6</dateGroupIndices>
+            <filePattern>.*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+        </dataSet>
     </category>
 </archive>

edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archiver/purger/RAW_DATA.xml

@@ -24,43 +24,60 @@
  * Date         Ticket#    Engineer    Description
  * ============ ========== =========== ==========================
  * Jun 20, 2013 1966       rferrel     Initial creation
+ * Aug 05, 2012 2224       rferrel     Changes to add dataSet tags.
  * 
  * @author rferrel
  * @version 1.0
 -->
     <!-- 
-    The <archive> contains five types of tags:
+    The <archive> contains four types of tags:
         <name>           - Required tag. The id for the archive such as Raw or Processed.
                            Used in the GUIs to filter which archive to display.
         <rootDir>        - Required tag. The root directory on the edex server for the archive.
         <minRetentionHours> - Required tag. The default number of hours to retain data in the <rootDir>
         <category>       - Logical grouping of archive's sub-directories.
     
-    The <category> has six types of tags:
+    The <category> has four types of tags:
         <name>              - Required tag. The id for the category such as grib2 or redbook.
                               Used in the GUIs to filter what is the display in the table.
         <extRetentionHours>    - Optional tag. The  extended retentionHours for selected directories of this category.
                               Default is the archive's <minRetentionHours>.
-        <dirPattern>        - Required tag. A regex pattern for finding directories for this category.
+        <dataSet>           - Required to have a least one. Each one contains a set of tags explained below.
+        <selectedDisplayNames>  - A directory matching <dirPattern>. These are selected directories from the Retention GUI. The purger
+                                  will used the category's <extRetentionHours> instead of the Arhivie's <minRetentionHours>.
+                                  An optional tag; may have more then one.(NOTE these are set internally when a selection configuration
+                                  file is loaded. They should not appear in the configuration file.)
+     The <dataSet> contains ? types of tags:
+        <dirPattern>        - Required to have at least one. A regex pattern for finding directories for this category.
                               The pattern is relative to the archive's <rootDir>. Wildcard patterns do not cross directory
                               delimiter /. Thus to match 3 levels of directories you would need .*/.*/.* .
-                              See patterns and groups section. There may be more then one of these tags. The restriction is they
+                              See patterns and groups section. There may be more then one of these tags in a <dataSet>. The
-                              must all have the same number of groupings and be in the same order.
+                              restriction is they must all have the same number of groupings and be in the same order to match up
+                              with the <displayLabel>'s, and <dateGroupIndicies>'s values.
         <filePattern>       - Optional tag.  A pattern to find files in the directories that match the <dirPattern>.
                               Default is everything in the directories that match <dirPattern>.
                               See patterns and groups section.
         <displayLabel>      - Required tag. The label to display for directories that match <dirPattern>. Any group in the
                               <dirPattern> may be made part of the label by placing the group's index inside parenthesis, {1}.
-                              More then one directory may match the <dirPattern>. The archive GUIs may  collapse them into a
+                              More then one directory may match the <dirPattern>. The archive GUIs may collapse them into a
                               single table entry.
-        <dateGroupIndicies> - Required tag. A comma separated list of 4 numbers which are the index of the groups in
+        <timeType>          - Optional tag to determine what type of time stamp is being used to get files/directories for retention
-                              <dirPattern> and/or <filePattern> that contain in order the year, mouth, day and hour information.
+                              and case creation. The value dictates how many groupings in the <dirPattern>s and/or <filePattern> are
+                              used to get the time stamp for a file. The four values are:
+                              Date - (default) the time stamp is made up of four groups in the patterns: year, month, day and hour.
+                              EpochSec - The time stamp has one group in the patterns which is the epoch time in seconds.
+                              EpochMS - The time stamp has one group in the patterns which is the epoch time in milliseconds.
+                              File - No group is used to get the time stamp. Instead use the files date of last modification.
+        <dateGroupIndicies> - Required tag when <timeType> has any value but File. 
+                              Date - A comma separated list of 4 numbers which are in order the index for year, month, day and hour.
+                              EpochSec - A number which is the index for the epoch in seconds.
+                              EpochMS - A number which is the index for the epoch in milliseconds.
+                              File - Not needed since no group is used to get the time stamp.
+                              
                               This is used to determine what files/directories to retain or a range of directories/files to copy
                               for case creation. Note to get the group's index the <dirPattern> and <filePattern> are combined.
                               Thus if there are 5 groups in the <dirPattern> then the first group in the <filePattern> is index 6.
-        <selectedDisplayNames>  - A directory matching <dirPattern>. These are selected directories from the Retention GUI. The purger
+     
-                                  will used the category's <extRetentionHours> instead of the Arhivie's <minRetentionHours>.
-                                  An optional tag; may have more then one.
      Patterns and groups.
      The <dirPattern> and <filePattern> use Java regex expressions; similar to the ldm's pqact.conf file.
      For more details see http://docs.oracle.com/javase/tutorial/essential/regex/
@@ -77,7 +94,7 @@
      Finally the sixth group will match any sub-directory that in the hour directory.
      
      Thus the directory paths <rootPath>/grib2/20130527/18/GFS will generate the display string, grib2 - GFS, and from the grouping we
-     can find the year, 201; month, 05; day, 27 and hour, 18.
+     can find the year, 2013; month, 05; day, 27 and hour, 18.
      
      Example with <filePattern>:
         <dirPattern>hdf5/(redbook)</dirPattern>
@@ -89,11 +106,14 @@
      <category>
         <name>Observation</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>(acars|airep|airmet|taf)</dirPattern>
+        <dataSet>
-        <dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
+           <dirPattern>(acars|airep|airmet|taf)</dirPattern>
-        <displayLabel>{1}</displayLabel>
+           <dirPattern>(bufrsigwx|sfcobs)/.*</dirPattern>
-        <dateGroupIndices>2,3,4,5</dateGroupIndices>
+           <displayLabel>{1}</displayLabel>
-        <filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+           <timeType>Date</timeType>
+           <dateGroupIndices>2,3,4,5</dateGroupIndices>
+           <filePattern>[^/]*-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+        </dataSet>
      </category>
      The first <dirPattern> looks for files matching the <filePattern> in the directories acars, airep, airmet or taf.
      The second <dirPattern> expects to find the files in subdirectories of bufrsigwx or sfcobs such as bufrsigwx/SWH.
@@ -109,54 +129,78 @@
     <name>Raw</name>
     <rootDir>/data_store/</rootDir>
     <minRetentionHours>24</minRetentionHours>
+    <category>
+        <name>Local</name>
+        <extRetentionHours>168</extRetentionHours>
+        <dataSet>
+            <dirPattern>(manual)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
+            <displayLabel>{1}</displayLabel>
+            <dateGroupIndices>2,3,4,5</dateGroupIndices>
+        </dataSet>
+    </category>
     <category>
         <name>Model</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>(grib|grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
+        <dataSet>
-        <displayLabel>{1} - {6}</displayLabel>
+            <dirPattern>(grib|grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
-        <dateGroupIndices>2,3,4,5</dateGroupIndices>
+            <displayLabel>{1} - {6}</displayLabel>
-    </category>
+            <dateGroupIndices>2,3,4,5</dateGroupIndices>
-    <category>
+        </dataSet>
-        <name>ModelSounding</name>
+        <dataSet>
-        <extRetentionHours>168</extRetentionHours>
+            <dirPattern>(bufrmos|modelsounding)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
-        <dirPattern>(bufrmos|modelsounding)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
+            <displayLabel>{1}</displayLabel>
-        <displayLabel>{1}</displayLabel>
+            <dateGroupIndices>2,3,4,5</dateGroupIndices>
-        <dateGroupIndices>2,3,4,5</dateGroupIndices>
+        </dataSet>
-    </category>
-    <category>
-        <name>Misc</name>
-        <extRetentionHours>168</extRetentionHours>
-        <dirPattern>(binlightning|climate|cwa|lsr|manual|misc_adm_messages|redbook|shef|summaries|svrwx|tcg|tcs|text|wwa|xml)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
-        <displayLabel>{1}</displayLabel>
-        <dateGroupIndices>2,3,4,5</dateGroupIndices>
     </category>
     <category>
         <name>Observation</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>(airep|airmet|bufrascat|bufrhdw|bufrmthdw|bufrncwf|bufrsigwx|bufrssmi|convsigmet|fire_wx_spot_fcst_reports|forecast|fssobs|intlsigmet|MAROB|maritime|metar|misc_sfc_obs|nonconvsigmet|pirep|sfcobs|synoptic)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
+        <dataSet>
-        <dirPattern>acars/(.*)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
+            <dirPattern>(airep|binlightning|bufrascat|bufrhdw|bufrmthdw|bufrssmi|lsr|MAROB|maritime|metar|misc_sfc_obs|pirep|sfcobs|shef|svrwx|synoptic)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
-        <displayLabel>{1}</displayLabel>
+            <displayLabel>{1}</displayLabel>
-        <dateGroupIndices>2,3,4,5</dateGroupIndices>
+            <dateGroupIndices>2,3,4,5</dateGroupIndices>
+        </dataSet>
+        <dataSet>
+            <dirPattern>(airep|binlightning|bufrascat|bufrhdw|bufrmthdw|bufrssmi|idft|lsr|MAROB|maritime|metar|misc_sfc_obs|pirep|sfcobs|shef|svrwx|synoptic)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
+            <dirPattern>(acars)/(.*)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
+            <displayLabel>{1} - {2}</displayLabel>
+            <dateGroupIndices>3,4,5,6</dateGroupIndices>
+        </dataSet>
+    </category>
+    <category>
+        <name>Products</name>
+        <extRetentionHours>168</extRetentionHours>
+        <dataSet>
+            <dirPattern>(airmet|bufrncwf|bufrsigwx|climate|convsigmet|cwa|fire_wx_spot_fcst_reports|forecast|intlsigmet|misc_adm_messages|nonconvsigmet|redbook|summaries|tcg|tcs|text|wwa|xml)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
+            <displayLabel>{1}</displayLabel>
+            <dateGroupIndices>2,3,4,5</dateGroupIndices>
+        </dataSet>
     </category>
     <category>
         <name>radar</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>radar/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
+        <dataSet>
-        <displayLabel>{5}</displayLabel>
+            <dirPattern>radar/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
-        <dateGroupIndices>1,2,3,4</dateGroupIndices>
+            <displayLabel>{5}</displayLabel>
+            <dateGroupIndices>1,2,3,4</dateGroupIndices>
+        </dataSet>
     </category>
     <category>
         <name>Satellite</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>sat/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
+        <dataSet>
-        <displayLabel>{5}</displayLabel>
+            <dirPattern>sat/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern>
-        <dateGroupIndices>1,2,3,4</dateGroupIndices>
+            <displayLabel>{5}</displayLabel>
+            <dateGroupIndices>1,2,3,4</dateGroupIndices>
+        </dataSet>
     </category>
     <category>
-        <name>Sounding</name>
+        <name>Profiles</name>
         <extRetentionHours>168</extRetentionHours>
-        <dirPattern>(acarssounding|bufrua|goessounding|poessounding|profiler|raobs|upperair)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
+        <dataSet>
-        <displayLabel>{1}</displayLabel>
+            <dirPattern>(bufrua|goessounding|poessounding|profiler|raobs|upperair)/(\d{4})(\d{2})(\d{2})/(\d{2})</dirPattern>
-        <dateGroupIndices>2,3,4,5</dateGroupIndices>
+            <displayLabel>{1}</displayLabel>
+            <dateGroupIndices>2,3,4,5</dateGroupIndices>
+        </dataSet>
     </category>
  </archive>