xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #2267 Refactored SbnSimulator into the sbnSimulator that runs on the wfo boxes and a centralSbnSimulator that runs on the centralRegistry box." into development

Browse source 
Former-commit-id: bfc87a0607 [formerly fe45d2b7e6] [formerly bfc87a0607 [formerly fe45d2b7e6] [formerly b6ffed0066 [formerly ef99656b07406288bf419037b05e85a5cf5771d6]]]
Former-commit-id: b6ffed0066
Former-commit-id: 5baa99e9c9 [formerly 4765eaefad]
Former-commit-id: f1d5148c49
This commit is contained in:
Richard Peter 2013-10-29 12:15:35 -05:00 • committed by Gerrit Code Review
commit 5ee9b1ce9d
8 changed files with 213 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
edexOsgi/build.edex/esb/conf/modes.xml
View file

@ -305,9 +305,16 @@
<exclude>.*datadelivery-ncf.*</exclude>
</mode>
<mode name="sbnSimulator">
<include>.*sbn-simulator.*</include>
<include>.*sbn-simulator-wfo.*</include>
<include>event-common.xml</include>
<include>eventbus-common.xml</include>
<exclude>.*sbn-simulator-ncf.*</exclude>
</mode>
<mode name="centralSbnSimulator">
<include>.*sbn-simulator-ncf.*</include>
<include>event-common.xml</include>
<include>eventbus-common.xml</include>
<exclude>.*sbn-simulator-wfo.*</exclude>
</mode>
<mode name="grib">
<include>grib-decode.xml</include>

39
edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/FileUtil.java
View file

@ -31,7 +31,13 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.FileChannel;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
@ -59,6 +65,7 @@ import java.util.zip.GZIPOutputStream;
* Mar 11, 2013 1645 djohnson Added file modification watcher.
* Mar 14, 2013 1794 djohnson FileUtil.listFiles now returns List.
* May 16, 2013 1966 rferrel Add sizeOfDirectory and listDirFiles method.
* Oct 18, 2013 2267 bgonzale Add listPaths method.
*
* </pre>
*
@ -925,4 +932,36 @@ public class FileUtil {
return size;
}
/**
* List files/directories that match a FileFilter.
*
* @param directory
* @param filter
* @param recurse
* @return
* @throws IOException
*/
public static List<Path> listPaths(File directory,
DirectoryStream.Filter<? super Path> filter, boolean recurse)
throws IOException {
// List of files / directories
List<Path> files = new LinkedList<Path>();
// Get files / directories in the directory accepted by the filter.
Path dirPath = FileSystems.getDefault().getPath(
directory.getAbsolutePath());
DirectoryStream<Path> stream = null;
try {
stream = Files.newDirectoryStream(dirPath, filter);
for (final Iterator<Path> it = stream.iterator(); it.hasNext();) {
files.add(it.next());
}
} finally {
if (stream != null) {
stream.close();
}
}
return files;
}
}

36
edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/file/FilenameFilters.java
View file

@ -21,6 +21,10 @@ package com.raytheon.uf.common.util.file;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
/**
* Consolidates common filename filters.
@ -32,6 +36,7 @@ import java.io.FilenameFilter;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 14, 2013 1794 djohnson Initial creation
* Oct 18, 2013 2267 bgonzale Added Path filters.
*
* </pre>
*
@ -154,7 +159,7 @@ public final class FilenameFilters {
};
/**
* Accepts directories.
* Accepts files.
*/
public static final FilenameFilter ACCEPT_FILES = new FilenameFilter() {
@Override
@ -162,6 +167,35 @@ public final class FilenameFilters {
return new File(dir.getPath(), name).isFile();
}
};
/**
* Accepts files.
*/
public static final FilenameFilter ACCEPT_VISIBLE_FILES = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
File tmp = new File(dir.getPath(), name);
return tmp.isFile() && !tmp.isHidden();
}
};
/**
* Accepts Path directories.
*/
public static final DirectoryStream.Filter<Path> ACCEPT_PATH_DIRECTORIES = new DirectoryStream.Filter<Path>() {
public boolean accept(Path path) throws IOException {
return (Files.isDirectory(path));
}
};
/**
* Accepts Path files.
*/
public static final DirectoryStream.Filter<Path> ACCEPT_PATH_FILES = new DirectoryStream.Filter<Path>() {
public boolean accept(Path path) throws IOException {
return (Files.isRegularFile(path));
}
};
/**
* Returns a {@link FilenameFilter} that matches the specified file

1
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth.ncf/META-INF/MANIFEST.MF
View file

@ -17,3 +17,4 @@ Require-Bundle: com.raytheon.uf.common.datadelivery.bandwidth;bundle-version="1.
com.google.guava;bundle-version="1.0.0",
com.raytheon.uf.edex.core;bundle-version="1.12.1174",
com.raytheon.uf.common.registry.ebxml;bundle-version="1.0.0"
Import-Package: com.raytheon.edex.site

30
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth.ncf/res/spring/sbn-simulator-ncf.xml Normal file
View file

@ -0,0 +1,30 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
<bean id="sbnSimulator"
class="com.raytheon.uf.edex.datadelivery.bandwidth.sbn.SbnSimulator" />
<camelContext id="SbnSimulator-context"
xmlns="http://camel.apache.org/schema/spring" errorHandlerRef="errorHandler">
<endpoint id="checkForSbnDataCron"
uri="quartz://datadelivery/sbnSimulator?cron=${sbnSimulator-checkForSbnData.cron}" />
<route id="distributeToSiteDirs">
<from uri="checkForSbnDataCron" />
<doTry>
<pipeline>
<bean ref="sbnSimulator" method="distributeToSiteDirs" />
</pipeline>
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:SbnSimulator" />
</doCatch>
</doTry>
</route>
</camelContext>
</beans>

0
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth.ncf/res/spring/sbn-simulator.xml → edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth.ncf/res/spring/sbn-simulator-wfo.xml
View file

63
edexOsgi/com.raytheon.uf.edex.datadelivery.bandwidth.ncf/src/com/raytheon/uf/edex/datadelivery/bandwidth/sbn/SbnSimulator.java
View file

@ -21,9 +21,13 @@ package com.raytheon.uf.edex.datadelivery.bandwidth.sbn;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.List;
import com.google.common.annotations.VisibleForTesting;
import com.raytheon.edex.site.SiteUtil;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.util.FileUtil;
@ -41,6 +45,7 @@ import com.raytheon.uf.edex.core.EDEXUtil;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 14, 2013 1648 djohnson Initial creation
* Oct 18, 2013 2267 bgonzale Added distribution to and check in site specific directories.
*
* </pre>
*
@ -84,6 +89,10 @@ public class SbnSimulator {
private final File directoryToScan;
private final File sitesDirectory;
private final File localSiteDirectory;
private final IFileProcessor fileProcessor;
/**
@ -91,7 +100,7 @@ public class SbnSimulator {
*/
public SbnSimulator() {
this(new File(System.getProperty("sbn.retrieval.transfer.directory")),
new CopyFileToManualIngest());
new CopyFileToManualIngest(), SiteUtil.getSite());
}
/**
@ -100,9 +109,12 @@ public class SbnSimulator {
* @param fileProcessor
*/
@VisibleForTesting
SbnSimulator(File scanDirectory, IFileProcessor fileProcessor) {
SbnSimulator(File scanDirectory, IFileProcessor fileProcessor, String site) {
this.fileProcessor = fileProcessor;
this.directoryToScan = scanDirectory;
this.sitesDirectory = new File(directoryToScan, "sbnSimulator");
this.localSiteDirectory = new File(sitesDirectory, site);
this.localSiteDirectory.mkdirs();
}
/**
@ -111,11 +123,11 @@ public class SbnSimulator {
* @throws IOException
*/
public void checkForSbnData() throws IOException {
final List<File> files = FileUtil.listFiles(directoryToScan,
FilenameFilters.ACCEPT_FILES, false);
final List<File> files = FileUtil.listFiles(localSiteDirectory,
FilenameFilters.ACCEPT_VISIBLE_FILES, false);
statusHandler
.info("Found [" + files.size() + "] files from the SBN...");
statusHandler.info("Found [" + files.size() + "] files for "
+ SiteUtil.getSite() + " from the SBN...");
for (File file : files) {
@ -131,4 +143,43 @@ public class SbnSimulator {
}
}
/**
* Distribute to the site directories. Enables all site client registries to
* ingest shared data.
*
* @throws IOException
*/
public void distributeToSiteDirs() throws IOException {
final List<Path> undistributedFiles = FileUtil.listPaths(
directoryToScan,
FilenameFilters.ACCEPT_PATH_FILES, false);
// get list of site dirs
final List<Path> sites = FileUtil.listPaths(sitesDirectory,
FilenameFilters.ACCEPT_PATH_DIRECTORIES, false);
statusHandler.info("Found [" + undistributedFiles.size() + "] files to distribute...");
// distribute to site specific directories
for (Path file : undistributedFiles) {
statusHandler.info("Distributing file [" + file + "]");
for (Path siteDir : sites) {
Path dest = FileSystems.getDefault().getPath(
siteDir.toString(), file.getFileName().toString());
Path hiddenDest = FileSystems.getDefault()
.getPath(siteDir.toString(),
"." + file.getFileName().toString());
// move to site sbn directory as hidden
java.nio.file.Files.copy(file, hiddenDest,
StandardCopyOption.REPLACE_EXISTING);
// rename dest to un-hidden
java.nio.file.Files.move(hiddenDest, dest,
StandardCopyOption.ATOMIC_MOVE);
statusHandler.info("===> to file [" + dest + "]");
}
// delete source file
java.nio.file.Files.delete(file);
}
}
}

64
tests/unit/com/raytheon/uf/edex/datadelivery/bandwidth/sbn/SbnSimulatorTest.java
View file

@ -19,8 +19,6 @@
**/
package com.raytheon.uf.edex.datadelivery.bandwidth.sbn;
import static com.raytheon.uf.common.util.Matchers.hasNoFiles;
import static org.junit.Assert.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@ -31,10 +29,13 @@ import static org.mockito.Mockito.verifyZeroInteractions;
import java.io.File;
import java.io.IOException;
import junit.framework.Assert;
import org.junit.Test;
import com.raytheon.uf.common.util.FileUtil;
import com.raytheon.uf.common.util.TestUtil;
import com.raytheon.uf.common.util.file.FilenameFilters;
import com.raytheon.uf.edex.datadelivery.bandwidth.sbn.SbnSimulator.IFileProcessor;
/**
@ -47,6 +48,7 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.sbn.SbnSimulator.IFileProcess
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 18, 2013 1648 djohnson Initial creation
* Oct 18, 2013 2267 bgonzale Updated tests to work with sbnSimulator reading from site specific dirs.
*
* </pre>
*
@ -55,26 +57,31 @@ import com.raytheon.uf.edex.datadelivery.bandwidth.sbn.SbnSimulator.IFileProcess
*/
public class SbnSimulatorTest {
private final static String SITE = "OAX";
private final File testDir = TestUtil
.setupTestClassDir(SbnSimulatorTest.class);
private final IFileProcessor fileProcessor = mock(IFileProcessor.class);
private final SbnSimulator simulator = new SbnSimulator(testDir,
fileProcessor);
fileProcessor, SITE);
@Test
public void processesEachFileInDirectory() throws IOException {
File fileOne = new File(testDir, "fileOne.txt");
File fileTwo = new File(testDir, "fileTwo.txt");
SbnSimulator simAAA = new SbnSimulator(testDir, fileProcessor, "AAA");
SbnSimulator simBBB = new SbnSimulator(testDir, fileProcessor, "BBB");
FileUtil.bytes2File("fileOne".getBytes(), fileOne);
FileUtil.bytes2File("fileTwo".getBytes(), fileTwo);
File fileOneExpectedResult = createTestFileGetExpected(testDir,
"fileOne.txt", SITE);
File fileTwoExpectedResult = createTestFileGetExpected(testDir,
"fileTwo.txt", "BBB");
simulator.checkForSbnData();
simAAA.checkForSbnData();
simBBB.checkForSbnData();
verify(fileProcessor).processFile(fileOne);
verify(fileProcessor).processFile(fileTwo);
verify(fileProcessor).processFile(fileOneExpectedResult);
verify(fileProcessor).processFile(fileTwoExpectedResult);
}
@Test
@ -86,32 +93,34 @@ public class SbnSimulatorTest {
@Test
public void fileInDirectoryProcessedOnlyOnce() throws IOException {
File fileOne = new File(testDir, "fileOne.txt");
FileUtil.bytes2File("fileOne".getBytes(), fileOne);
File fileOneExpectedResult = createTestFileGetExpected(testDir,
"fileOne.txt", SITE);
simulator.checkForSbnData();
verify(fileProcessor, times(1)).processFile(fileOne);
verify(fileProcessor, times(1)).processFile(fileOneExpectedResult);
}
@Test
public void fileInDirectoryIsDeleted() throws IOException {
File fileOne = new File(testDir, "fileOne.txt");
FileUtil.bytes2File("fileOne".getBytes(), fileOne);
createTestFileGetExpected(testDir, "fileOne.txt", SITE);
simulator.checkForSbnData();
assertThat(testDir, hasNoFiles());
int fileCount = FileUtil.listFiles(testDir,
FilenameFilters.ACCEPT_FILES, true).size();
Assert.assertEquals(
"Found unexpected files in the processing directory.", 0,
fileCount);
}
@Test
public void errorOnOneFileDoesNotStopTheOthers() throws IOException {
FileUtil.bytes2File("fileOne".getBytes(), new File(testDir,
"fileOne.txt"));
FileUtil.bytes2File("fileTwo".getBytes(), new File(testDir,
"fileTwo.txt"));
File fileOneExpectedResult = createTestFileGetExpected(testDir,
"fileOne.txt", "AA1");
File fileTwoExpectedResult = createTestFileGetExpected(testDir,
"fileTwo.txt", "AA2");
doThrow(new IOException()).when(fileProcessor).processFile(
any(File.class));
@ -120,4 +129,17 @@ public class SbnSimulatorTest {
verify(fileProcessor, times(2)).processFile(any(File.class));
}
private File createTestFileGetExpected(File dir, String name, String site)
throws IOException {
File file = new File(dir, name);
File fileExpectedResult = new File(dir, "sbnSimulator/" + site
+ File.separator + name);
FileUtil.bytes2File(name.getBytes(), file);
simulator.distributeToSiteDirs();
return fileExpectedResult;
}
}