Merge "Issue #2267 Refactored SbnSimulator into the sbnSimulator that runs on the wfo boxes and a centralSbnSimulator that runs on the centralRegistry box." into development

Former-commit-id: bfc87a0607 [formerly fe45d2b7e6] [formerly bfc87a0607 [formerly fe45d2b7e6] [formerly b6ffed0066 [formerly ef99656b07406288bf419037b05e85a5cf5771d6]]]
Former-commit-id: b6ffed0066
Former-commit-id: 5baa99e9c9 [formerly 4765eaefad]
Former-commit-id: f1d5148c49

edexOsgi/build.edex/esb/conf/modes.xml

@@ -305,9 +305,16 @@
         <exclude>.*datadelivery-ncf.*</exclude>
     </mode>
     <mode name="sbnSimulator">
-        <include>.*sbn-simulator.*</include>
+        <include>.*sbn-simulator-wfo.*</include>
         <include>event-common.xml</include>
         <include>eventbus-common.xml</include>
+        <exclude>.*sbn-simulator-ncf.*</exclude>
+    </mode>
+    <mode name="centralSbnSimulator">
+        <include>.*sbn-simulator-ncf.*</include>
+        <include>event-common.xml</include>
+        <include>eventbus-common.xml</include>
+        <exclude>.*sbn-simulator-wfo.*</exclude>
     </mode>
 	<mode name="grib">
 		<include>grib-decode.xml</include>

edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/FileUtil.java

@@ -31,7 +31,13 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.channels.FileChannel;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
@@ -59,6 +65,7 @@ import java.util.zip.GZIPOutputStream;
  * Mar 11, 2013 1645       djohnson    Added file modification watcher.
  * Mar 14, 2013 1794       djohnson    FileUtil.listFiles now returns List.
  * May 16, 2013 1966       rferrel     Add sizeOfDirectory and listDirFiles method.
+ * Oct 18, 2013 2267       bgonzale    Add listPaths method.
  * 
  * </pre>
  * 
@@ -925,4 +932,36 @@ public class FileUtil {
         return size;
     }
 
+    /**
+     * List files/directories that match a FileFilter.
+     * 
+     * @param directory
+     * @param filter
+     * @param recurse
+     * @return
+     * @throws IOException
+     */
+    public static List<Path> listPaths(File directory,
+            DirectoryStream.Filter<? super Path> filter, boolean recurse)
+            throws IOException {
+        // List of files / directories
+        List<Path> files = new LinkedList<Path>();
+
+        // Get files / directories in the directory accepted by the filter.
+        Path dirPath = FileSystems.getDefault().getPath(
+                directory.getAbsolutePath());
+        DirectoryStream<Path> stream = null;
+        try {
+            stream = Files.newDirectoryStream(dirPath, filter);
+            for (final Iterator<Path> it = stream.iterator(); it.hasNext();) {
+                files.add(it.next());
+            }
+        } finally {
+            if (stream != null) {
+                stream.close();
+            }
+        }
+        return files;
+    }
+
 }

edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/file/FilenameFilters.java

@@ -21,6 +21,10 @@ package com.raytheon.uf.common.util.file;
 
 import java.io.File;
 import java.io.FilenameFilter;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
 
 /**
  * Consolidates common filename filters.
@@ -32,6 +36,7 @@ import java.io.FilenameFilter;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Mar 14, 2013 1794       djohnson     Initial creation
+ * Oct 18, 2013 2267       bgonzale     Added Path filters.
  * 
  * </pre>
  * 
@@ -154,7 +159,7 @@ public final class FilenameFilters {
     };
 
     /**
-     * Accepts directories.
+     * Accepts files.
      */
     public static final FilenameFilter ACCEPT_FILES = new FilenameFilter() {
         @Override
@@ -163,6 +168,35 @@ public final class FilenameFilters {
         }
     };
     
+    /**
+     * Accepts files.
+     */
+    public static final FilenameFilter ACCEPT_VISIBLE_FILES = new FilenameFilter() {
+        @Override
+        public boolean accept(File dir, String name) {
+            File tmp = new File(dir.getPath(), name);
+            return tmp.isFile() && !tmp.isHidden();
+        }
+    };
+
+    /**
+     * Accepts Path directories.
+     */
+    public static final DirectoryStream.Filter<Path> ACCEPT_PATH_DIRECTORIES = new DirectoryStream.Filter<Path>() {
+        public boolean accept(Path path) throws IOException {
+            return (Files.isDirectory(path));
+        }
+    };
+
+    /**
+     * Accepts Path files.
+     */
+    public static final DirectoryStream.Filter<Path> ACCEPT_PATH_FILES = new DirectoryStream.Filter<Path>() {
+        public boolean accept(Path path) throws IOException {
+            return (Files.isRegularFile(path));
+        }
+    };
+
     /**
      * Returns a {@link FilenameFilter} that matches the specified file
      * extension.

edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth.ncf/META-INF/MANIFEST.MF

@@ -17,3 +17,4 @@ Require-Bundle: com.raytheon.uf.common.datadelivery.bandwidth;bundle-version="1.
  com.google.guava;bundle-version="1.0.0",
  com.raytheon.uf.edex.core;bundle-version="1.12.1174",
  com.raytheon.uf.common.registry.ebxml;bundle-version="1.0.0"
+Import-Package: com.raytheon.edex.site

edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth.ncf/res/spring/sbn-simulator-ncf.xml

@@ -0,0 +1,30 @@
+<beans xmlns="http://www.springframework.org/schema/beans"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
+    http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
+
+    <bean id="sbnSimulator"
+        class="com.raytheon.uf.edex.datadelivery.bandwidth.sbn.SbnSimulator" />
+
+    <camelContext id="SbnSimulator-context"
+        xmlns="http://camel.apache.org/schema/spring" errorHandlerRef="errorHandler">
+
+        <endpoint id="checkForSbnDataCron"
+            uri="quartz://datadelivery/sbnSimulator?cron=${sbnSimulator-checkForSbnData.cron}" />
+
+        <route id="distributeToSiteDirs">
+            <from uri="checkForSbnDataCron" />
+            <doTry>
+                <pipeline>
+                    <bean ref="sbnSimulator" method="distributeToSiteDirs" />
+                </pipeline>
+                <doCatch>
+                    <exception>java.lang.Throwable</exception>
+                    <to
+                        uri="log:SbnSimulator" />
+                </doCatch>
+            </doTry>
+        </route>
+    </camelContext>
+
+</beans>

edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth.ncf/src/com/raytheon/uf/edex/datadelivery/bandwidth/sbn/SbnSimulator.java

@@ -21,9 +21,13 @@ package com.raytheon.uf.edex.datadelivery.bandwidth.sbn;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.List;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.raytheon.edex.site.SiteUtil;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.util.FileUtil;
@@ -41,6 +45,7 @@ import com.raytheon.uf.edex.core.EDEXUtil;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Mar 14, 2013 1648       djohnson     Initial creation
+ * Oct 18, 2013 2267       bgonzale     Added distribution to and check in site specific directories.
  * 
  * </pre>
  * 
@@ -84,6 +89,10 @@ public class SbnSimulator {
 
     private final File directoryToScan;
 
+    private final File sitesDirectory;
+
+    private final File localSiteDirectory;
+
     private final IFileProcessor fileProcessor;
 
     /**
@@ -91,7 +100,7 @@ public class SbnSimulator {
      */
     public SbnSimulator() {
         this(new File(System.getProperty("sbn.retrieval.transfer.directory")),
-                new CopyFileToManualIngest());
+                new CopyFileToManualIngest(), SiteUtil.getSite());
     }
 
     /**
@@ -100,9 +109,12 @@ public class SbnSimulator {
      * @param fileProcessor
      */
     @VisibleForTesting
-    SbnSimulator(File scanDirectory, IFileProcessor fileProcessor) {
+    SbnSimulator(File scanDirectory, IFileProcessor fileProcessor, String site) {
         this.fileProcessor = fileProcessor;
         this.directoryToScan = scanDirectory;
+        this.sitesDirectory = new File(directoryToScan, "sbnSimulator");
+        this.localSiteDirectory = new File(sitesDirectory, site);
+        this.localSiteDirectory.mkdirs();
     }
 
     /**
@@ -111,11 +123,11 @@ public class SbnSimulator {
      * @throws IOException
      */
     public void checkForSbnData() throws IOException {
-        final List<File> files = FileUtil.listFiles(directoryToScan,
+        final List<File> files = FileUtil.listFiles(localSiteDirectory,
-                FilenameFilters.ACCEPT_FILES, false);
+                FilenameFilters.ACCEPT_VISIBLE_FILES, false);
 
-        statusHandler
+        statusHandler.info("Found [" + files.size() + "] files for "
-                .info("Found [" + files.size() + "] files from the SBN...");
+                + SiteUtil.getSite() + " from the SBN...");
 
         for (File file : files) {
 
@@ -131,4 +143,43 @@ public class SbnSimulator {
         }
     }
 
+    /**
+     * Distribute to the site directories. Enables all site client registries to
+     * ingest shared data.
+     * 
+     * @throws IOException
+     */
+    public void distributeToSiteDirs() throws IOException {
+        final List<Path> undistributedFiles = FileUtil.listPaths(
+                directoryToScan,
+                FilenameFilters.ACCEPT_PATH_FILES, false);
+        // get list of site dirs
+        final List<Path> sites = FileUtil.listPaths(sitesDirectory,
+                FilenameFilters.ACCEPT_PATH_DIRECTORIES, false);
+        
+        statusHandler.info("Found [" + undistributedFiles.size() + "] files to distribute...");
+        
+        // distribute to site specific directories
+        for (Path file : undistributedFiles) {
+            statusHandler.info("Distributing file [" + file + "]");
+            for (Path siteDir : sites) {
+                Path dest = FileSystems.getDefault().getPath(
+                        siteDir.toString(), file.getFileName().toString());
+                Path hiddenDest = FileSystems.getDefault()
+                        .getPath(siteDir.toString(),
+                                "." + file.getFileName().toString());
+
+                // move to site sbn directory as hidden
+                java.nio.file.Files.copy(file, hiddenDest,
+                        StandardCopyOption.REPLACE_EXISTING);
+                // rename dest to un-hidden
+                java.nio.file.Files.move(hiddenDest, dest,
+                        StandardCopyOption.ATOMIC_MOVE);
+                statusHandler.info("===> to file [" + dest + "]");
+            }
+            // delete source file
+            java.nio.file.Files.delete(file);
+        }
+    }
+
 }

tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/sbn/SbnSimulatorTest.java

@@ -19,8 +19,6 @@
  **/
 package com.raytheon.uf.edex.datadelivery.bandwidth.sbn;
 
-import static com.raytheon.uf.common.util.Matchers.hasNoFiles;
-import static org.junit.Assert.assertThat;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
@@ -31,10 +29,13 @@ import static org.mockito.Mockito.verifyZeroInteractions;
 import java.io.File;
 import java.io.IOException;
 
+import junit.framework.Assert;
+
 import org.junit.Test;
 
 import com.raytheon.uf.common.util.FileUtil;
 import com.raytheon.uf.common.util.TestUtil;
+import com.raytheon.uf.common.util.file.FilenameFilters;
 import com.raytheon.uf.edex.datadelivery.bandwidth.sbn.SbnSimulator.IFileProcessor;
 
 /**
@@ -47,6 +48,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.sbn.SbnSimulator.IFileProcess
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Mar 18, 2013 1648       djohnson     Initial creation
+ * Oct 18, 2013 2267       bgonzale     Updated tests to work with sbnSimulator reading from site specific dirs.
  * 
  * </pre>
  * 
@@ -55,26 +57,31 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.sbn.SbnSimulator.IFileProcess
  */
 public class SbnSimulatorTest {
 
+    private final static String SITE = "OAX";
     private final File testDir = TestUtil
             .setupTestClassDir(SbnSimulatorTest.class);
 
     private final IFileProcessor fileProcessor = mock(IFileProcessor.class);
 
     private final SbnSimulator simulator = new SbnSimulator(testDir,
-            fileProcessor);
+            fileProcessor, SITE);
 
     @Test
     public void processesEachFileInDirectory() throws IOException {
-        File fileOne = new File(testDir, "fileOne.txt");
+        SbnSimulator simAAA = new SbnSimulator(testDir, fileProcessor, "AAA");
-        File fileTwo = new File(testDir, "fileTwo.txt");
+        SbnSimulator simBBB = new SbnSimulator(testDir, fileProcessor, "BBB");
 
-        FileUtil.bytes2File("fileOne".getBytes(), fileOne);
+        File fileOneExpectedResult = createTestFileGetExpected(testDir,
-        FileUtil.bytes2File("fileTwo".getBytes(), fileTwo);
+                "fileOne.txt", SITE);
+        File fileTwoExpectedResult = createTestFileGetExpected(testDir,
+                "fileTwo.txt", "BBB");
 
         simulator.checkForSbnData();
+        simAAA.checkForSbnData();
+        simBBB.checkForSbnData();
 
-        verify(fileProcessor).processFile(fileOne);
+        verify(fileProcessor).processFile(fileOneExpectedResult);
-        verify(fileProcessor).processFile(fileTwo);
+        verify(fileProcessor).processFile(fileTwoExpectedResult);
     }
 
     @Test
@@ -86,32 +93,34 @@ public class SbnSimulatorTest {
 
     @Test
     public void fileInDirectoryProcessedOnlyOnce() throws IOException {
-        File fileOne = new File(testDir, "fileOne.txt");
+        File fileOneExpectedResult = createTestFileGetExpected(testDir,
-
+                "fileOne.txt", SITE);
-        FileUtil.bytes2File("fileOne".getBytes(), fileOne);
 
         simulator.checkForSbnData();
 
-        verify(fileProcessor, times(1)).processFile(fileOne);
+        verify(fileProcessor, times(1)).processFile(fileOneExpectedResult);
     }
 
     @Test
     public void fileInDirectoryIsDeleted() throws IOException {
-        File fileOne = new File(testDir, "fileOne.txt");
+        createTestFileGetExpected(testDir, "fileOne.txt", SITE);
-
-        FileUtil.bytes2File("fileOne".getBytes(), fileOne);
 
         simulator.checkForSbnData();
 
-        assertThat(testDir, hasNoFiles());
+        int fileCount = FileUtil.listFiles(testDir,
+                FilenameFilters.ACCEPT_FILES, true).size();
+
+        Assert.assertEquals(
+                "Found unexpected files in the processing directory.", 0,
+                fileCount);
     }
 
     @Test
     public void errorOnOneFileDoesNotStopTheOthers() throws IOException {
-        FileUtil.bytes2File("fileOne".getBytes(), new File(testDir,
+        File fileOneExpectedResult = createTestFileGetExpected(testDir,
-                "fileOne.txt"));
+                "fileOne.txt", "AA1");
-        FileUtil.bytes2File("fileTwo".getBytes(), new File(testDir,
+        File fileTwoExpectedResult = createTestFileGetExpected(testDir,
-                "fileTwo.txt"));
+                "fileTwo.txt", "AA2");
 
         doThrow(new IOException()).when(fileProcessor).processFile(
                 any(File.class));
@@ -120,4 +129,17 @@ public class SbnSimulatorTest {
 
         verify(fileProcessor, times(2)).processFile(any(File.class));
     }
+
+    private File createTestFileGetExpected(File dir, String name, String site)
+            throws IOException {
+        File file = new File(dir, name);
+        File fileExpectedResult = new File(dir, "sbnSimulator/" + site
+                + File.separator + name);
+
+        FileUtil.bytes2File(name.getBytes(), file);
+        simulator.distributeToSiteDirs();
+        return fileExpectedResult;
+    }
+
+
 }