From 589be2735150d972baebd2da6f9f668856938e8e Mon Sep 17 00:00:00 2001 From: Roger Ferrel Date: Thu, 23 May 2013 14:08:59 -0500 Subject: [PATCH] Issue #1966 Changes to CategoryConfig to make parsing more efficent. Change-Id: I3fc9bf03eccea8ae63cb37a4469e512c1fac9e87 Former-commit-id: 00ed24f0812dfadee0b632531355083a4fcb6e1b [formerly 24f1c70168724751217f3fc808402dd83ff7125f] [formerly 365af6b7ce5d7c1a62234232ad851834a567e2cd] [formerly 365af6b7ce5d7c1a62234232ad851834a567e2cd [formerly cbee73144717370bcc63c163a889c1ee30773137]] [formerly 9263b54845a1e7bed8ea2622f69fbb63fd909e4d [formerly 365af6b7ce5d7c1a62234232ad851834a567e2cd [formerly cbee73144717370bcc63c163a889c1ee30773137] [formerly 9263b54845a1e7bed8ea2622f69fbb63fd909e4d [formerly d8a510b87d026d57eb282583c4cf2f786749dc95]]]] Former-commit-id: 9263b54845a1e7bed8ea2622f69fbb63fd909e4d Former-commit-id: c9d911bbb8ab90460a3b74ae86b587cc4b8858b4 [formerly e443885c0974b7dc457a2ea698d05b926b780848] [formerly 838eacc802c5e72f7ed28822627b986b8c97d454 [formerly fce5c2ca1e2e6444e9f1ac4faf07ccaa6baa3a36]] Former-commit-id: 838eacc802c5e72f7ed28822627b986b8c97d454 Former-commit-id: 618feeda6d3519b2d2b27c216a9d133d8e24d5f0 --- .../uf/viz/archive/data/FileInfo.java | 3 +- cots/org.apache.commons.io/.classpath | 2 +- .../META-INF/MANIFEST.MF | 5 +- cots/org.apache.commons.io/build.properties | 4 +- .../com.raytheon.uf.common.archive/.classpath | 8 +- .../META-INF/MANIFEST.MF | 4 +- .../common/archive/config/ArchiveConfig.java | 7 +- .../archive/config/ArchiveConfigManager.java | 485 ++++++++++++++---- .../common/archive/config/CategoryConfig.java | 78 ++- .../archive/file/FileArchiveElement.java | 3 - .../base/archive/PROCESSED_DATA.xml | 96 +++- .../common_static/base/archive/RAW_DATA.xml | 108 +++- 12 files changed, 605 insertions(+), 198 deletions(-) diff --git a/cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/data/FileInfo.java b/cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/data/FileInfo.java index 6be0cfc1df..d656b96d90 100644 --- a/cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/data/FileInfo.java +++ b/cave/com.raytheon.uf.viz.archive/src/com/raytheon/uf/viz/archive/data/FileInfo.java @@ -32,7 +32,8 @@ import org.eclipse.core.runtime.jobs.Job; import com.raytheon.uf.common.util.FileUtil; /** - * This class uses a obtains information on a File in a Job in order to remove from the UI thread. + * This class uses a obtains information on a File in a Job in order to remove + * from the UI thread. * * 
  * 
diff --git a/cots/org.apache.commons.io/.classpath b/cots/org.apache.commons.io/.classpath
index b11d5ff541..ea1f089f85 100644
--- a/cots/org.apache.commons.io/.classpath
+++ b/cots/org.apache.commons.io/.classpath
@@ -2,7 +2,7 @@
 
 	
 	
-	
+	
 	
 	
 	
diff --git a/cots/org.apache.commons.io/META-INF/MANIFEST.MF b/cots/org.apache.commons.io/META-INF/MANIFEST.MF
index 8476542db3..20e9dd30f5 100644
--- a/cots/org.apache.commons.io/META-INF/MANIFEST.MF
+++ b/cots/org.apache.commons.io/META-INF/MANIFEST.MF
@@ -3,9 +3,7 @@ Bundle-ManifestVersion: 2
 Bundle-Name: Io
 Bundle-SymbolicName: org.apache.commons.io
 Bundle-Version: 2.4
-Bundle-ClassPath: commons-io-2.4.jar,
- commons-io-2.4-javadoc.jar,
- commons-io-2.4-sources.jar
+Bundle-ClassPath: commons-io-2.4.jar
 Bundle-Vendor: Apache
 Export-Package: org.apache.commons.io,
  org.apache.commons.io.comparator,
@@ -14,3 +12,4 @@ Export-Package: org.apache.commons.io,
  org.apache.commons.io.monitor,
  org.apache.commons.io.output
 Bundle-RequiredExecutionEnvironment: JavaSE-1.6
+Bundle-ActivationPolicy: lazy
diff --git a/cots/org.apache.commons.io/build.properties b/cots/org.apache.commons.io/build.properties
index d8a3f3f1f6..0c4fd61bb6 100644
--- a/cots/org.apache.commons.io/build.properties
+++ b/cots/org.apache.commons.io/build.properties
@@ -1,4 +1,2 @@
 bin.includes = META-INF/,\
-               commons-io-2.4.jar,\
-               commons-io-2.4-javadoc.jar,\
-               commons-io-2.4-sources.jar
+               commons-io-2.4.jar
diff --git a/edexOsgi/com.raytheon.uf.common.archive/.classpath b/edexOsgi/com.raytheon.uf.common.archive/.classpath
index 692796b71b..ad32c83a78 100644
--- a/edexOsgi/com.raytheon.uf.common.archive/.classpath
+++ b/edexOsgi/com.raytheon.uf.common.archive/.classpath
@@ -1,11 +1,7 @@
 
 
-	
 	
-	
-	
-	
-	
-	
+	
+	
 	
 
diff --git a/edexOsgi/com.raytheon.uf.common.archive/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.common.archive/META-INF/MANIFEST.MF
index 37386310a6..52c587c7da 100644
--- a/edexOsgi/com.raytheon.uf.common.archive/META-INF/MANIFEST.MF
+++ b/edexOsgi/com.raytheon.uf.common.archive/META-INF/MANIFEST.MF
@@ -11,4 +11,6 @@ Export-Package: com.raytheon.uf.common.archive,
  com.raytheon.uf.common.archive.file
 Require-Bundle: com.raytheon.uf.common.util;bundle-version="1.12.1174",
  com.raytheon.uf.common.localization;bundle-version="1.12.1174",
- com.raytheon.uf.common.status;bundle-version="1.12.1174"
+ com.raytheon.uf.common.status;bundle-version="1.12.1174",
+ org.apache.commons.io;bundle-version="2.4.0",
+ com.raytheon.uf.common.time;bundle-version="1.12.1174"
diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfig.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfig.java
index a70efc389f..a012f3f864 100644
--- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfig.java
+++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfig.java
@@ -27,8 +27,6 @@ import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 
-import org.junit.Assert;
-
 /**
  * Archive data information for retention, purging and archiving. An example:
  * 
@@ -115,7 +113,6 @@ public class ArchiveConfig implements Comparable {
      * @param name
      */
     public void setName(String name) {
-        Assert.assertNotNull(name);
         this.name = name;
     }
 
@@ -135,12 +132,11 @@ public class ArchiveConfig implements Comparable {
      * @param rootDir
      */
     public void setRootDir(String rootDir) {
-        Assert.assertNotNull(rootDir);
         this.rootDir = rootDir;
     }
 
     /**
-     * Retrive the archive's retention hours.
+     * Retrieve the archive's retention hours.
      * 
      * @return retentionHours
      */
@@ -154,7 +150,6 @@ public class ArchiveConfig implements Comparable {
      * @param retentionHours
      */
     public void setRetentionHours(int retentionHours) {
-        Assert.assertTrue(retentionHours > 0);
         this.retentionHours = retentionHours;
     }
 
diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java
index 384e698f91..c72bf0893c 100644
--- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java
+++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java
@@ -24,27 +24,33 @@ import java.io.FileFilter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.text.FieldPosition;
+import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TimeZone;
 import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import javax.xml.bind.JAXB;
 
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.io.filefilter.IOFileFilter;
+import org.apache.commons.io.filefilter.RegexFileFilter;
+
 import com.raytheon.uf.common.localization.IPathManager;
-import com.raytheon.uf.common.localization.LocalizationContext;
-import com.raytheon.uf.common.localization.LocalizationContext.LocalizationLevel;
-import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType;
 import com.raytheon.uf.common.localization.LocalizationFile;
 import com.raytheon.uf.common.localization.PathManagerFactory;
 import com.raytheon.uf.common.localization.exception.LocalizationException;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
+import com.raytheon.uf.common.status.UFStatus.Priority;
+import com.raytheon.uf.common.time.util.TimeUtil;
 import com.raytheon.uf.common.util.FileUtil;
 
 /**
@@ -64,39 +70,33 @@ import com.raytheon.uf.common.util.FileUtil;
  * @version 1.0
  */
 public class ArchiveConfigManager {
-    private final static ArchiveConfigManager instance = new ArchiveConfigManager();
-
     private final IUFStatusHandler statusHandler = UFStatus
             .getHandler(ArchiveConfigManager.class);
 
+    private final static ArchiveConfigManager instance = new ArchiveConfigManager();
+
     public final String ARCHIVE_DIR = "archive";
 
     private IPathManager pathMgr;
 
     private final Map archiveMap = new HashMap();
 
-    private DirectoryFilter directoryFilter = new DirectoryFilter();
-
-    private Pattern subPatterns[] = new Pattern[] { Pattern.compile("\\1"),
-            Pattern.compile("\\2"), Pattern.compile("\\3"),
-            Pattern.compile("\\4"), Pattern.compile("\\5"),
-            Pattern.compile("\\6"), Pattern.compile("\\7"),
-            Pattern.compile("\\8"), Pattern.compile("\\9"), };
-
-    private Pattern yearPattern = Pattern.compile("$\\{YYYY}");
-
-    private Pattern monthPattern = Pattern.compile("$\\{MM}");
-
-    private Pattern dayPattern = Pattern.compile("$\\{DD}");
-
-    private Pattern hourPattern = Pattern.compile("$\\{HH}");
+    private Map>>> displayDirMap = new HashMap>>>();
 
     public final static ArchiveConfigManager getInstance() {
         return instance;
     }
 
+    /**
+     * Private constructor for singleton.
+     */
     private ArchiveConfigManager() {
         pathMgr = PathManagerFactory.getPathManager();
+        try {
+            reset();
+        } catch (Exception e) {
+            statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e);
+        }
     }
 
     /**
@@ -105,51 +105,33 @@ public class ArchiveConfigManager {
      * @return archiveConfigList
      */
     private LocalizationFile[] getArchiveConfigFiles() {
-
-        Map confFileMap = new HashMap();
-        LocalizationContext ctx = pathMgr.getContext(
-                LocalizationType.COMMON_STATIC, LocalizationLevel.BASE);
-
-        LocalizationFile[] files = pathMgr.listFiles(ctx, ARCHIVE_DIR,
+        LocalizationFile[] files = pathMgr.listStaticFiles(ARCHIVE_DIR,
                 new String[] { ".xml" }, false, true);
-
-        for (LocalizationFile lf : files) {
-            confFileMap.put(lf.getName().trim(), lf);
-        }
-
-        ctx = pathMgr.getContext(LocalizationType.COMMON_STATIC,
-                LocalizationLevel.SITE);
-
-        files = pathMgr.listFiles(ctx, ARCHIVE_DIR, new String[] { ".xml" },
-                false, true);
-        for (LocalizationFile lf : files) {
-            confFileMap.put(lf.getName().trim(), lf);
-        }
-
-        files = new LocalizationFile[confFileMap.size()];
-        int i = 0;
-        for (String key : confFileMap.keySet()) {
-            files[i] = confFileMap.get(key);
-            ++i;
-        }
         return files;
     }
 
-    public String[] getArchiveDataNamesList() throws IOException,
-            LocalizationException {
-        if (archiveMap.size() == 0) {
-            LocalizationFile[] files = getArchiveConfigFiles();
-            for (LocalizationFile lFile : files) {
-                ArchiveConfig archiveConfig = unmarshalArhiveConfigFromXmlFile(lFile);
-                archiveMap.put(archiveConfig.getName(), archiveConfig);
-            }
-        }
+    /**
+     * Get a sorted list of the archive data names.
+     * 
+     * @return names
+     */
+    public String[] getArchiveDataNamesList() {
         String[] names = archiveMap.keySet().toArray(new String[0]);
         Arrays.sort(names, 0, names.length, String.CASE_INSENSITIVE_ORDER);
 
         return names;
     }
 
+    /**
+     * Obtain the names of the categories for the named archive data.
+     * 
+     * @param archiveConfigName
+     * @return categoryNames
+     */
+    public String[] getCategoryNames(String archiveConfigName) {
+        return getCategoryNames(archiveMap.get(archiveConfigName));
+    }
+
     /**
      * @param archiveConfig
      * @return
@@ -166,38 +148,286 @@ public class ArchiveConfigManager {
         return names;
     }
 
+    /**
+     * Load the archiveConfig information form the localized file.
+     * 
+     * @param lFile
+     * @return archiveConfig
+     * @throws IOException
+     * @throws LocalizationException
+     */
     public ArchiveConfig loadArchiveData(LocalizationFile lFile)
             throws IOException, LocalizationException {
         return unmarshalArhiveConfigFromXmlFile(lFile);
     }
 
+    /**
+     * Save the Archive Configuration data to the localized file.
+     * 
+     * @param lFile
+     * @param archiveConfig
+     * @throws IOException
+     * @throws LocalizationException
+     */
     public void saveArchiveConfig(LocalizationFile lFile,
-            ArchiveConfig archiveConfig) throws IOException,
-            LocalizationException {
-        marshalArchiveConfigToXmlFile(archiveConfig, lFile);
+            ArchiveConfig archiveConfig) {
+        try {
+            marshalArchiveConfigToXmlFile(archiveConfig, lFile);
+        } catch (Exception e) {
+            statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), e);
+        }
     }
 
-    private void marshalArchiveConfigToXmlFile(ArchiveConfig archiveConfig,
-            LocalizationFile lFile) throws IOException, LocalizationException {
-        OutputStream stream = null;
-        try {
-            stream = lFile.openOutputStream();
-            JAXB.marshal(archiveConfig, stream);
-        } finally {
-            if (stream != null) {
-                try {
-                    stream.close();
-                } catch (IOException ex) {
-                    // ignore
+    /**
+     * Add the file entry to the desired display map.
+     * 
+     * @param file
+     * @param archiveConfig
+     * @param categoryConfig
+     * @param displayLabel
+     */
+    private void addDirToDisplayMap(File file, ArchiveConfig archiveConfig,
+            CategoryConfig categoryConfig, String displayLabel) {
+        Map>> cMap = displayDirMap
+                .get(archiveConfig);
+        if (cMap == null) {
+            cMap = new HashMap>>();
+            displayDirMap.put(archiveConfig, cMap);
+        }
+
+        Map> dirMap = cMap.get(categoryConfig);
+        if (dirMap == null) {
+            dirMap = new HashMap>();
+            cMap.put(categoryConfig, dirMap);
+        }
+
+        List fileList = dirMap.get(displayLabel);
+        if (fileList == null) {
+            fileList = new ArrayList();
+            dirMap.put(displayLabel, fileList);
+        }
+        fileList.add(file);
+    }
+
+    /**
+     * Get a list of directories associated with the display label.
+     * 
+     * @param archiveConfig
+     * @param categoryConfig
+     * @param displayLabel
+     * @return
+     */
+    private File[] getDirFromDisplayMap(ArchiveConfig archiveConfig,
+            CategoryConfig categoryConfig, String displayLabel) {
+        Map>> cMap = displayDirMap
+                .get(archiveConfig);
+
+        File[] result = new File[0];
+        if (cMap == null) {
+            return result;
+        }
+
+        Map> dirMap = cMap.get(categoryConfig);
+        if (dirMap == null) {
+            return result;
+        }
+
+        List fileList = dirMap.get(displayLabel);
+        if (fileList == null) {
+            return result;
+        }
+
+        return fileList.toArray(result);
+    }
+
+    /**
+     * Clear maps and reloads the maps from the configuration information.
+     * 
+     * @throws IOException
+     * @throws LocalizationException
+     */
+    public void reset() throws IOException, LocalizationException {
+        clearDisplayMap();
+        archiveMap.clear();
+        LocalizationFile[] files = getArchiveConfigFiles();
+        for (LocalizationFile lFile : files) {
+            ArchiveConfig archiveConfig = unmarshalArhiveConfigFromXmlFile(lFile);
+            archiveMap.put(archiveConfig.getName(), archiveConfig);
+        }
+    }
+
+    /**
+     * Walk the display directory maps clearing all entries.
+     */
+    private void clearDisplayMap() {
+        for (ArchiveConfig archiveConfig : displayDirMap.keySet()) {
+            Map>> cMap = displayDirMap
+                    .get(archiveConfig);
+            for (CategoryConfig categoryConfig : cMap.keySet()) {
+                Map> dirMap = cMap.get(categoryConfig);
+                dirMap.remove(categoryConfig);
+                for (String displayLabel : dirMap.keySet()) {
+                    List fileList = dirMap.get(displayLabel);
+                    fileList.clear();
                 }
+                dirMap.clear();
             }
         }
     }
 
+    /**
+     * Get a list of directories/files for the desired display label bounded by
+     * the start and end time inclusive.
+     * 
+     * @param archiveName
+     *            - name of the ArchiveConfig containing the category
+     * @param categoryName
+     *            - name of the CategoryConfig containing the display
+     * @param displayLabel
+     *            - the display label
+     * @param startCal
+     *            - Start time if null epoch time is used
+     * @param endCal
+     *            - End time if null current simulated time is used.
+     * @return files
+     */
     public File[] getDisplayFiles(String archiveName, String categoryName,
-            List dispalyList, Calendar startTime, Calendar endTime) {
-        // TODO implement
-        return null;
+            String displayLabel, Calendar startCal, Calendar endCal) {
+        long startTime = 0L;
+        if (startCal != null) {
+            // Set to beginning of the hour
+            Calendar cal = (Calendar) startCal.clone();
+            cal.set(Calendar.MILLISECOND, 0);
+            cal.set(Calendar.MINUTE, 0);
+            cal.set(Calendar.SECOND, 0);
+            startTime = cal.getTimeInMillis();
+        }
+
+        Calendar cal = null;
+        if (endCal != null) {
+            cal = (Calendar) endCal.clone();
+        } else {
+            cal = TimeUtil.newCalendar();
+        }
+
+        // Set to start of the next hour.
+        cal.set(Calendar.MILLISECOND, 0);
+        cal.set(Calendar.SECOND, 0);
+        cal.set(Calendar.MINUTE, 0);
+        cal.add(Calendar.HOUR_OF_DAY, 1);
+
+        long endTime = cal.getTimeInMillis();
+        return getDisplayFiles(archiveName, categoryName, displayLabel,
+                startTime, endTime);
+    }
+
+    /**
+     * Get all of the files/directories associated with the display label.
+     * 
+     * @param archiveName
+     * @param categoryName
+     * @param displayLabel
+     * @return files
+     */
+    public File[] getDisplayFiles(String archiveName, String categoryName,
+            String displayLabel) {
+        return getDisplayFiles(archiveName, categoryName, displayLabel, null,
+                null);
+    }
+
+    /**
+     * Get the files/directories associated with the display label with time
+     * stamps between inclusive start time and exclusive end time.
+     * 
+     * @param archiveName
+     * @param categoryName
+     * @param displayLabel
+     * @param startTime
+     * @param endTime
+     * @return files
+     */
+    private File[] getDisplayFiles(String archiveName, String categoryName,
+            String displayLabel, long startTime, long endTime) {
+        ArchiveConfig archiveConfig = archiveMap.get(archiveName);
+        CategoryConfig categoryConfig = findCategory(archiveConfig,
+                categoryName);
+
+        String[] indexValues = categoryConfig.getDateGroupIndices().split(
+                "\\s*,\\s*");
+        int yearIndex = Integer.parseInt(indexValues[0]);
+        int monthIndex = Integer.parseInt(indexValues[1]);
+        int dayIndex = Integer.parseInt(indexValues[2]);
+        int hourIndex = Integer.parseInt(indexValues[3]);
+
+        String filePatternStr = categoryConfig.getFilePattern();
+
+        boolean dirOnly = (filePatternStr == null)
+                || ".*".equals(filePatternStr);
+
+        File dirs[] = getDirFromDisplayMap(archiveConfig, categoryConfig,
+                displayLabel);
+
+        int beginIndex = archiveConfig.getRootDir().length();
+
+        Calendar fileCal = TimeUtil.newCalendar();
+        fileCal.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+        List fileList = new ArrayList();
+
+        if (dirOnly) {
+            Pattern pattern = Pattern.compile(categoryConfig.getDirPattern());
+
+            for (File dir : dirs) {
+                String path = dir.getAbsolutePath().substring(beginIndex);
+                Matcher matcher = pattern.matcher(path);
+                if (matcher.matches()) {
+                    int year = Integer.parseInt(matcher.group(yearIndex));
+                    // Adjust month value to Calendar's 0 - 11
+                    int month = Integer.parseInt(matcher.group(monthIndex)) - 1;
+                    int day = Integer.parseInt(matcher.group(dayIndex));
+                    int hour = Integer.parseInt(matcher.group(hourIndex));
+                    fileCal.set(year, month, day, hour, 0, 0);
+                    long fileTime = fileCal.getTimeInMillis();
+                    if ((startTime <= fileTime) && (fileTime < endTime)) {
+                        fileList.add(dir);
+                    }
+                }
+            }
+        } else {
+            Pattern pattern = Pattern.compile(categoryConfig.getDirPattern()
+                    + File.separator + categoryConfig.getFilePattern());
+            final Pattern filePattern = Pattern.compile("^" + filePatternStr
+                    + "$");
+            for (File dir : dirs) {
+                List fList = FileUtil.listDirFiles(dir, new FileFilter() {
+
+                    @Override
+                    public boolean accept(File pathname) {
+                        return filePattern.matcher(pathname.getName())
+                                .matches();
+                    }
+                }, false);
+                for (File file : fList) {
+                    String path = file.getAbsolutePath().substring(beginIndex);
+                    Matcher matcher = pattern.matcher(path);
+                    if (matcher.matches()) {
+                        int year = Integer.parseInt(matcher.group(yearIndex));
+                        // Adjust month value to Calendar's 0 - 11
+                        int month = Integer.parseInt(matcher.group(monthIndex)) - 1;
+                        int day = Integer.parseInt(matcher.group(dayIndex));
+                        int hour = Integer.parseInt(matcher.group(hourIndex));
+                        fileCal.set(year, month, day, hour, 0, 0);
+                        long fileTime = fileCal.getTimeInMillis();
+                        if ((startTime <= fileTime) && (fileTime < endTime)) {
+                            fileList.add(dir);
+                        }
+                    }
+                }
+            }
+        }
+
+        File[] files = fileList.toArray(new File[0]);
+        return files;
     }
 
     /**
@@ -211,43 +441,67 @@ public class ArchiveConfigManager {
      */
     public String[] getDisplayLabels(String archiveName, String categoryName) {
         ArchiveConfig archiveConfig = archiveMap.get(archiveName);
-        CategoryConfig category = findCategory(archiveConfig, categoryName);
+        CategoryConfig categoryConfig = findCategory(archiveConfig,
+                categoryName);
+        String dirPattern = categoryConfig.getDirPattern();
 
         File rootFile = new File(archiveConfig.getRootDir());
 
-        List dirList = findDirs(rootFile);
+        String[] subExpr = dirPattern.split(File.separator);
+        List dirs = new ArrayList();
+        dirs.add(rootFile);
+        List tmpDirs = new ArrayList();
+        List swpDirs = null;
 
-        Pattern pattern = Pattern.compile(category.getDirPattern());
+        for (String regex : subExpr) {
+            Pattern subPattern = Pattern.compile("^" + regex + "$");
+            IOFileFilter filter = FileFilterUtils
+                    .makeDirectoryOnly(new RegexFileFilter(subPattern));
+
+            for (File dir : dirs) {
+                File[] list = dir.listFiles();
+                if (list != null) {
+                    List dirList = Arrays.asList(list);
+                    tmpDirs.addAll(Arrays.asList(FileFilterUtils.filter(filter,
+                            dirList)));
+                }
+            }
+            swpDirs = dirs;
+            dirs = tmpDirs;
+            tmpDirs = swpDirs;
+            tmpDirs.clear();
+        }
+
+        // index for making directory paths' relative to the root path.
+        int beginIndex = rootFile.getAbsolutePath().length() + 1;
+        Pattern pattern = Pattern.compile("^" + dirPattern + "$");
         TreeSet displays = new TreeSet(
                 String.CASE_INSENSITIVE_ORDER);
 
+        MessageFormat msgfmt = new MessageFormat(categoryConfig.getDisplay());
         StringBuffer sb = new StringBuffer();
-        for (File dir : dirList) {
-            Matcher matcher = pattern.matcher(dir.getAbsolutePath());
-            if (matcher.matches()) {
-                String display = category.getDisplay();
-                int groupCnt = matcher.groupCount();
+        FieldPosition pos0 = new FieldPosition(0);
 
-                for (int i = 0; i < groupCnt; ++i) {
-                    Matcher subMatcher = subPatterns[i].matcher(display);
-                    sb.setLength(0);
-                    while (subMatcher.find()) {
-                        subMatcher.appendReplacement(sb, matcher.group(i));
-                    }
-                    subMatcher.appendTail(sb);
-                    display = sb.toString();
+        for (File file : dirs) {
+            String path = file.getAbsolutePath().substring(beginIndex);
+            Matcher matcher = pattern.matcher(path);
+            if (matcher.matches()) {
+                sb.setLength(0);
+                String[] args = new String[matcher.groupCount() + 1];
+                args[0] = matcher.group();
+                for (int i = 1; i < args.length; ++i) {
+                    args[i] = matcher.group(i);
                 }
-                displays.add(display);
+                String displayLabel = msgfmt.format(args, sb, pos0).toString();
+                addDirToDisplayMap(file, archiveConfig, categoryConfig,
+                        displayLabel);
+                displays.add(displayLabel);
             }
         }
 
         return displays.toArray(new String[0]);
     }
 
-    private List findDirs(File parentDir) {
-        return FileUtil.listDirFiles(parentDir, directoryFilter, true);
-    }
-
     /**
      * Obtain the category with the given name.
      * 
@@ -265,6 +519,33 @@ public class ArchiveConfigManager {
         return null;
     }
 
+    /**
+     * This does the work of saving an instance of ArchiveConfig to a localized
+     * file.
+     * 
+     * @param archiveConfig
+     * @param lFile
+     * @throws IOException
+     * @throws LocalizationException
+     */
+    private void marshalArchiveConfigToXmlFile(ArchiveConfig archiveConfig,
+            LocalizationFile lFile) throws IOException, LocalizationException {
+        OutputStream stream = null;
+        stream = lFile.openOutputStream();
+        JAXB.marshal(archiveConfig, stream);
+        stream.close();
+        lFile.save();
+    }
+
+    /**
+     * This does the work of taking a localized file and returning a instance of
+     * the ArchiveConfig class from the data in the file.
+     * 
+     * @param lFile
+     * @return archiveConfig
+     * @throws IOException
+     * @throws LocalizationException
+     */
     private ArchiveConfig unmarshalArhiveConfigFromXmlFile(
             LocalizationFile lFile) throws IOException, LocalizationException {
         ArchiveConfig archiveConfig = null;
@@ -283,20 +564,4 @@ public class ArchiveConfigManager {
         }
         return archiveConfig;
     }
-
-    private class DirectoryFilter implements FileFilter {
-
-        /*
-         * (non-Javadoc)
-         * 
-         * @see java.io.FileFilter#accept(java.io.File)
-         */
-        @Override
-        public boolean accept(File pathname) {
-            if (pathname.isDirectory() && !pathname.getName().startsWith(".")) {
-                return true;
-            }
-            return false;
-        }
-    }
 }
diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java
index 418833d348..a23a609568 100644
--- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java
+++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/CategoryConfig.java
@@ -24,8 +24,6 @@ import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 
-import org.junit.Assert;
-
 /**
  * Information on a category for a given archive.
  * 
@@ -36,10 +34,10 @@ import org.junit.Assert;
  *   <name>redbook</name>
  *   <!-- When 0 default to the parent archive's retentionHours -->
  *   <retentionHours>0</retentionHours>
- *   <dirPattern>hdf5/redbook/([^/]*)/</dirPattern>
- *   <display>redbook - \1</display>
- *   <saveDir>hdf5/redbook/([^/]*)/</saveDir>
- *   <saveFiles>redbook-${YYYY}-${MM}-${DD}-${HH}\..*</saveFiles>
+ *   <dirPattern>hdf5/(redbook)</dirPattern>
+ *   <display>{1}</display>
+ *   <filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+ *   <dateGroupIndices>2,3,4,5</dateGroupIndices>
  * </category>
  *
* @@ -49,11 +47,9 @@ import org.junit.Assert; * <category> * <name>Model grib</name> * <retentionHours>0</retentionHours> - * <dirPattern>grib/[^/]*/[^/]*/(.*\)</dirPattern> - * <display>\1</display> - * <saveDir>grib/${YYYY}${MM}${DD}/${HH}/(.*)</saveDir> - * <saveFiles>.*</saveFiles> - * <selectList>grib/${YYYY}${MM}${DD}/${HH}/NCEP_QPF/</selectList> + * <dirPattern>grib/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirPattern> + * <display>{5}</display> + * <dateGroupIndices>1,2,3,4</dateGroupIndices> * </category> * * @@ -93,7 +89,7 @@ public class CategoryConfig implements Comparable { * example: * * 
-     * <dirPattern>grib2/[^/]*/[^/]*/(.*)/</dirPattern>
+     * <dirPattern>grib2/\d{8}/\d{2}/(.*)/</dirPattern>
      *
*/ @XmlElement(name = "dirPattern") @@ -104,26 +100,33 @@ public class CategoryConfig implements Comparable { * dirPatern may be displayed. For example: * * 
-     * <dirName>grib2/[^/]*/[^/]*/(.*)</dirName>
-     * <display>grib2 - \1</display>
+     * <dirName>(grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)</dirName>
+     * <display>{1} - {6}</display>
      *
* - * The \1 will be replaced with directory names 2 levels down from the grib2 - * directory. + * The {1} will be replaced by the first group (grib2) in the regex + * expression in dirName. The {6} is the sixth group (.*). {0} is the whole + * expression match. */ @XmlElement(name = "display") private String display; /** - * A pattern to find the directories to save. This may contain the year, - * month, day and hour information. For example: + * A comma separated list of 4 numbers representing the group indices + * specifying the location of the numeric date time stamp. The indices must + * be in the order of year, month, day and hour. The group numbering starts + * with the first group in the dirPattern and continues with any grouping in + * the filePattern. * * 
-     * <saveDir>grib2/${YYYY}${MM}{$DD}/${HH}/.*</saveDir>
+     *   <dirPattern>hdf5/(redbook)</dirPattern>
+     *   <display>{1}</display>
+     *   <filePattern>redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..*</filePattern>
+     *   <dateGroupIndices>2,3,4,5</dateGroupIndices>
      *
*/ - @XmlElement(name = "saveDir") - private String saveDir; + @XmlElement(name = "dateGroupIndices") + private String dateGroupIndices; /** * A saveDir directory may contain files with data for several days. This @@ -135,8 +138,8 @@ public class CategoryConfig implements Comparable { * <saveFile>redbook-${YYYY}-${MM}-${DD}-${HH}\..*<saveFiles> * */ - @XmlElement(name = "saveFiles") - private String saveFiles; + @XmlElement(name = "filePattern") + private String filePattern; /* * Constructor. @@ -158,7 +161,6 @@ public class CategoryConfig implements Comparable { * @param name */ public void setName(String name) { - Assert.assertNotNull(name); this.name = name; } @@ -175,7 +177,6 @@ public class CategoryConfig implements Comparable { * Set the retention hours must be greater then zero. */ public void setRetentionHours(int retentionHours) { - Assert.assertTrue(retentionHours > 0); this.retentionHours = retentionHours; } @@ -194,7 +195,6 @@ public class CategoryConfig implements Comparable { * @param dirPattern */ public void setDirPattern(String dirPattern) { - Assert.assertNotNull(dirPattern); this.dirPattern = dirPattern; } @@ -208,22 +208,21 @@ public class CategoryConfig implements Comparable { } /** - * Set the disaplay pattern. + * Set the display pattern. * * @param display */ public void setDisplay(String display) { - Assert.assertNotNull(display); this.display = display; } /** * Get the save directory pattern.. * - * @return savedir + * @return dateGroups */ - public String getSaveDir() { - return saveDir; + public String getDateGroupIndices() { + return dateGroupIndices; } /** @@ -231,9 +230,8 @@ public class CategoryConfig implements Comparable { * * @param saveDir */ - public void setSaveDir(String saveDir) { - Assert.assertNotNull(saveDir); - this.saveDir = saveDir; + public void setDateGroupIndices(String dateGroupIndices) { + this.dateGroupIndices = dateGroupIndices; } /** @@ -241,8 +239,8 @@ public class CategoryConfig implements Comparable { * * @return saveFiles */ - public String getSaveFiles() { - return saveFiles; + public String getFilePattern() { + return filePattern; } /** @@ -250,8 +248,8 @@ public class CategoryConfig implements Comparable { * * @param saveFiles */ - public void setSaveFiles(String saveFiles) { - this.saveFiles = saveFiles; + public void setFilePattern(String filePattern) { + this.filePattern = filePattern; } /* @@ -275,9 +273,9 @@ public class CategoryConfig implements Comparable { sb.append("Category [ name: ").append(getName()); sb.append(", retentionHours: ").append(getRetentionHours()); sb.append(", dirPattern: ").append(getDirPattern()); + sb.append(", filePattern: ").append(getFilePattern()); sb.append(", display: ").append(getDisplay()); - sb.append(", saveDir: ").append(getSaveDir()); - sb.append(", saveFiles: ").append(getSaveFiles()); + sb.append(", dateGroupIndices: ").append(getDateGroupIndices()); sb.append("]"); return sb.toString(); } diff --git a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/file/FileArchiveElement.java b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/file/FileArchiveElement.java index 3f613fc913..2490070076 100644 --- a/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/file/FileArchiveElement.java +++ b/edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/file/FileArchiveElement.java @@ -21,8 +21,6 @@ package com.raytheon.uf.common.archive.file; import java.io.File; -import org.junit.Assert; - import com.raytheon.uf.common.archive.IArchiveElement; import com.raytheon.uf.common.archive.exception.ArchiveException; @@ -48,7 +46,6 @@ public class FileArchiveElement implements IArchiveElement { private String name; public FileArchiveElement(String name) { - Assert.assertNotNull(name); this.name = name; } diff --git a/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archive/PROCESSED_DATA.xml b/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archive/PROCESSED_DATA.xml index 02b7807bdd..fc13e5e8ee 100644 --- a/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archive/PROCESSED_DATA.xml +++ b/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archive/PROCESSED_DATA.xml @@ -1,4 +1,84 @@ + + Processed /awips2/edex/data/archive/ @@ -6,17 +86,17 @@ redbook 0 - hdf5/redbook/([^/]*)/ - redbook - \1 - hdf5/redbook/([^/]*)/ - redbook-${YYYY}-${MM}-${DD}-${HH}\..* + hdf5/(redbook) + {1} + redbook-(\d{4})-(\d{2})-(\d{2})-(\d{2})\..* + 2,3,4,5 obs 0 - hdf5/(obs)/ - \1 - hdf5/(obs)/ - metar-${YYYY}-${MM}-${DD}-${HH}\.* + hdf5/(obs) + {1} + metar-(\d{4})-(\d{2})-(\d{2})-(\d{2})\.* + 2,3,4,5 diff --git a/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archive/RAW_DATA.xml b/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archive/RAW_DATA.xml index 11538cbd80..0c83694b93 100644 --- a/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archive/RAW_DATA.xml +++ b/edexOsgi/com.raytheon.uf.edex.archive/utility/common_static/base/archive/RAW_DATA.xml @@ -1,4 +1,84 @@ + + Raw /data_store/ @@ -6,33 +86,29 @@ Model grib 0 - grib/[^/]*/[^/]*/(.*\) - \1 - grib/${YYYY}${MM}${DD}/${HH}/(.*) - .* + grib/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*) + {5} + 1,2,3,4 Model grib2 0 - grib2/[^/]*/[^/]*/(.*\) - \1 - grib2/${YYYY}${MM}${DD}/${HH}/.* - .* + (grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*) + {1} - {6} + 2,3,4,5 Model other 0 - (acars|airmet|binlighting|bufrmos|bufrua|bufsigwx|convsigmet|goessndg|manual|mdlsndg|poessndg|profiler|shef|text)/[^/]*/[^/]*/(.*) - \1 - \2 - (acars|airmet|binlighting|bufrmos|bufrua|bufsigwx|convsigmet|goessndg|manual|mdlsndg|poessndg|profiler|shef|text)/YYYYMMDD/HH/.* - .* + (acars|airmet|binlightning|bufrmos|bufrua|bufrsigwx|convsigmet|goessndg|manual|mdlsndg|poessndg|profiler|shef|text)/(\d{4})(\d{2})(\d{2})/(\d{2}) + {1} + 2,3,4,5 Satellite 101 - sat/[^/]*/[^/]*/(.*) - \1 - sat/${YYYY}${MM}${DD}/${HH}/.* - .* + sat/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*) + {5} + 1,2,3,4