xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
awips2/edex/distributed-computing/index.html
srcarter3 d8dc7cbc93 deploy: 40474f474f
2023-12-14 21:49:45 +00:00

1342 lines
No EOL
39 KiB
HTML
Raw Blame History

<!DOCTYPE html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta http-equiv="x-ua-compatible" content="ie=edge">
<meta name="description" content="Documentation for all things AWIPS.">
<link rel="canonical" href="http://unidata.github.io/awips2/edex/distributed-computing/">
<meta name="author" content="support-awips@unidata.ucar.edu">
<link rel="shortcut icon" href="../../images/favicon.ico">
<meta name="generator" content="mkdocs-1.5.3, mkdocs-unidata-2.0.3">
<title>Distributed EDEX - NSF Unidata AWIPS Manual</title>
<script src="../../assets/javascripts/modernizr-e826f8942a.js"></script>
<link rel="stylesheet" href="../../assets/stylesheets/application-37fa87ea8f.css">
<link rel="stylesheet" href="../../assets/stylesheets/application-624922a329.palette.css">
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono">
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
<link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
<link rel="stylesheet" href="../../css/extra.css">
</head>
<body data-md-color-primary="blue-grey" data-md-color-accent="light-blue">
<svg class="md-svg">
<defs>
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
</defs>
</svg>
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="drawer">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="search">
<label class="md-overlay" data-md-component="overlay" for="drawer"></label>
<header class="md-header" data-md-component="header">
<a href="../..">
<div class="background-logo" style="background-image:url('../../images/cave.png');background-repeat:no-repeat;height:107px">
<nav class="md-header-nav md-grid">
<div class="md-flex">
<div class="md-flex__cell md-flex__cell--shrink">
<a href="http://unidata.github.io/awips2/" title="NSF Unidata AWIPS Manual" class="md-header-nav__button md-logo">
<img src="../../images/home.png" width="24" height="24">
</a>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<label class="md-icon md-icon--menu md-header-nav__button" for="drawer"></label>
</div>
<div class="md-flex__cell md-flex__cell--stretch">
<span class="md-flex__ellipsis md-header-nav__title">
<span class="md-header-nav__parent">
EDEX User Manual
</span>
Distributed EDEX
</span>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<label class="md-icon md-icon--search md-header-nav__button" for="search"></label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="search"></label>
<div class="md-search__inner">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" required placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query">
<label class="md-icon md-search__icon" for="search"></label>
<button type="reset" class="md-icon md-search__icon" data-md-component="reset">&#xE5CD;</button>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" data-md-scrollfix>
<div class="md-search-result" data-md-component="result" data-md-lang-search="" data-md-lang-tokenizer="[\s\-]+">
<div class="md-search-result__meta" data-md-lang-result-none="No matching documents" data-md-lang-result-one="1 matching document" data-md-lang-result-other="# matching documents">
Type to start searching
</div>
<ol class="md-search-result__list"></ol>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<div class="md-header-nav__source">
<a href="https://github.com/Unidata/awips2" title="Go to repository" class="md-source" data-md-source="github">
<div class="md-source__icon">
<svg viewBox="0 0 24 24" width="24" height="24">
<use xlink:href="#github" width="24" height="24"></use>
</svg>
</div>
<div class="md-source__repository">
GitHub
</div>
</a>
</div>
</div>
</div>
</nav>
</header>
<div class="md-container">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" data-md-level="0">
<label class="md-nav__title md-nav__title--site" for="drawer">
<a href="http://unidata.github.io/awips2/" title="NSF Unidata AWIPS Manual" class="md-nav__button md-logo">
<img src="../../images/home.png">
</a>
NSF Unidata AWIPS Manual
</label>
<div class="md-nav__source">
<a href="https://github.com/Unidata/awips2" title="Go to repository" class="md-source" data-md-source="github">
<div class="md-source__icon">
<svg viewBox="0 0 24 24" width="24" height="24">
<use xlink:href="#github" width="24" height="24"></use>
</svg>
</div>
<div class="md-source__repository">
GitHub
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../.." title="Home" class="md-nav__link">
Home
</a>
</li>
<li class="md-nav__item">
<a href="../../install/install-cave/" title="Install CAVE" class="md-nav__link">
Install CAVE
</a>
</li>
<li class="md-nav__item">
<a href="../../install/install-edex/" title="Install EDEX" class="md-nav__link">
Install EDEX
</a>
</li>
<li class="md-nav__item">
<a href="../../appendix/common-problems/" title="Common Problems" class="md-nav__link">
Common Problems
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-5" type="checkbox" id="nav-5">
<label class="md-nav__link" for="nav-5">
CAVE User Manual
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
<label class="md-nav__title" for="nav-5">
CAVE User Manual
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../cave/d2d-perspective/" title="D2D Perspective" class="md-nav__link">
D2D Perspective
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/maps-views-projections/" title="Maps, Views, Projections" class="md-nav__link">
Maps, Views, Projections
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/bundles-and-procedures/" title="Displays and Procedures" class="md-nav__link">
Displays and Procedures
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/localization-perspective/" title="Localization Perspective" class="md-nav__link">
Localization Perspective
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/nsharp/" title="NSHARP" class="md-nav__link">
NSHARP
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/warngen/" title="WarnGen Walkthrough" class="md-nav__link">
WarnGen Walkthrough
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-gis-shapefiles/" title="GIS and Shapefiles" class="md-nav__link">
GIS and Shapefiles
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-gridded-models/" title="Volume Browser" class="md-nav__link">
Volume Browser
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-all-tools/" title="Tools" class="md-nav__link">
Tools
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-edit-menus/" title="Editing Menus" class="md-nav__link">
Editing Menus
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/cave-localization/" title="Change Localization" class="md-nav__link">
Change Localization
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/import-export/" title="Import/Export" class="md-nav__link">
Import/Export
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-6" type="checkbox" id="nav-6" checked>
<label class="md-nav__link" for="nav-6">
EDEX User Manual
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
<label class="md-nav__title" for="nav-6">
EDEX User Manual
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../install/start-edex/" title="EDEX Basic Commands" class="md-nav__link">
EDEX Basic Commands
</a>
</li>
<li class="md-nav__item">
<a href="../settings/" title="EDEX Settings" class="md-nav__link">
EDEX Settings
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="toc">
<label class="md-nav__link md-nav__link--active" for="toc">
Distributed EDEX
</label>
<a href="./" title="Distributed EDEX" class="md-nav__link md-nav__link--active">
Distributed EDEX
</a>
<nav class="md-nav md-nav--secondary">
<label class="md-nav__title" for="toc">Table of contents</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#nsf-unidatas-current-edex-server" title="NSF Unidata's Current EDEX Server" class="md-nav__link">
NSF Unidata's Current EDEX Server
</a>
</li>
<li class="md-nav__item">
<a href="#example-installation" title="Example Installation" class="md-nav__link">
Example Installation
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#databaserequest-server" title="Database/Request Server" class="md-nav__link">
Database/Request Server
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#1-install" title="1. Install" class="md-nav__link">
1. Install
</a>
</li>
<li class="md-nav__item">
<a href="#2-iptables-config" title="2. IPtables Config" class="md-nav__link">
2. IPtables Config
</a>
</li>
<li class="md-nav__item">
<a href="#3-database-config" title="3. Database Config" class="md-nav__link">
3. Database Config
</a>
</li>
<li class="md-nav__item">
<a href="#4-start-edex" title="4. Start EDEX" class="md-nav__link">
4. Start EDEX
</a>
</li>
<li class="md-nav__item">
<a href="#5-monitor-services" title="5. Monitor Services" class="md-nav__link">
5. Monitor Services
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#ancillary-edex-server-ingestdecode-edex-server" title="Ancillary EDEX Server (Ingest/Decode EDEX Server)" class="md-nav__link">
Ancillary EDEX Server (Ingest/Decode EDEX Server)
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#1-install_1" title="1. Install" class="md-nav__link">
1. Install
</a>
</li>
<li class="md-nav__item">
<a href="#2-edex-config" title="2. EDEX Config" class="md-nav__link">
2. EDEX Config
</a>
</li>
<li class="md-nav__item">
<a href="#3-modify-the-edexservicelist" title="3. Modify the edexServiceList" class="md-nav__link">
3. Modify the edexServiceList
</a>
</li>
<li class="md-nav__item">
<a href="#4-configure-your-ldm" title="4. Configure your LDM" class="md-nav__link">
4. Configure your LDM
</a>
</li>
<li class="md-nav__item">
<a href="#5-start-edex" title="5. Start EDEX" class="md-nav__link">
5. Start EDEX
</a>
</li>
<li class="md-nav__item">
<a href="#4-monitor-services" title="4. Monitor Services" class="md-nav__link">
4. Monitor Services
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#additional-notes" title="Additional Notes" class="md-nav__link">
Additional Notes
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../edex-ingest-docker-container/" title="Docker EDEX" class="md-nav__link">
Docker EDEX
</a>
</li>
<li class="md-nav__item">
<a href="../ldm/" title="LDM Feeds" class="md-nav__link">
LDM Feeds
</a>
</li>
<li class="md-nav__item">
<a href="../data-distribution-files/" title="Data Distribution Files" class="md-nav__link">
Data Distribution Files
</a>
</li>
<li class="md-nav__item">
<a href="../new-grid/" title="Ingest a New Grid" class="md-nav__link">
Ingest a New Grid
</a>
</li>
<li class="md-nav__item">
<a href="../data-purge/" title="Purging and Retention" class="md-nav__link">
Purging and Retention
</a>
</li>
<li class="md-nav__item">
<a href="../edex-users/" title="Monitor Users" class="md-nav__link">
Monitor Users
</a>
</li>
<li class="md-nav__item">
<a href="../data-plugins/" title="Data Plugins" class="md-nav__link">
Data Plugins
</a>
</li>
<li class="md-nav__item">
<a href="../case-studies/" title="Archive Case Studies" class="md-nav__link">
Archive Case Studies
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../../python/overview/" title="Python-AWIPS" class="md-nav__link">
Python-AWIPS
</a>
</li>
<li class="md-nav__item">
<a href="../../appendix/educational-resources/" title="Educational Resources" class="md-nav__link">
Educational Resources
</a>
</li>
<li class="md-nav__item">
<a href="../../dev/awips-development-environment/" title="Development" class="md-nav__link">
Development
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-10" type="checkbox" id="nav-10">
<label class="md-nav__link" for="nav-10">
Appendix
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
<label class="md-nav__title" for="nav-10">
Appendix
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../appendix/appendix-grid-parameters/" title="AWIPS Grid Parameters" class="md-nav__link">
AWIPS Grid Parameters
</a>
</li>
<li class="md-nav__item">
<a href="../../appendix/maps-database/" title="Maps Database" class="md-nav__link">
Maps Database
</a>
</li>
<li class="md-nav__item">
<a href="../../appendix/appendix-acronyms/" title="Acronyms and Abbreviations" class="md-nav__link">
Acronyms and Abbreviations
</a>
</li>
<li class="md-nav__item">
<a href="../../appendix/appendix-wsr88d/" title="WSR-88D Product Table" class="md-nav__link">
WSR-88D Product Table
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/cave-keyboard-shortcuts/" title="Keyboard Shortcuts" class="md-nav__link">
Keyboard Shortcuts
</a>
</li>
<li class="md-nav__item">
<a href="../../raytheon/cave_d2d/" title="Raytheon: CAVE User's Manual" class="md-nav__link">
Raytheon: CAVE User's Manual
</a>
</li>
<li class="md-nav__item">
<a href="../../raytheon/smm/" title="Raytheon: AWIPS System Manager's Manual" class="md-nav__link">
Raytheon: AWIPS System Manager's Manual
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary">
<label class="md-nav__title" for="toc">Table of contents</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#nsf-unidatas-current-edex-server" title="NSF Unidata's Current EDEX Server" class="md-nav__link">
NSF Unidata's Current EDEX Server
</a>
</li>
<li class="md-nav__item">
<a href="#example-installation" title="Example Installation" class="md-nav__link">
Example Installation
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#databaserequest-server" title="Database/Request Server" class="md-nav__link">
Database/Request Server
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#1-install" title="1. Install" class="md-nav__link">
1. Install
</a>
</li>
<li class="md-nav__item">
<a href="#2-iptables-config" title="2. IPtables Config" class="md-nav__link">
2. IPtables Config
</a>
</li>
<li class="md-nav__item">
<a href="#3-database-config" title="3. Database Config" class="md-nav__link">
3. Database Config
</a>
</li>
<li class="md-nav__item">
<a href="#4-start-edex" title="4. Start EDEX" class="md-nav__link">
4. Start EDEX
</a>
</li>
<li class="md-nav__item">
<a href="#5-monitor-services" title="5. Monitor Services" class="md-nav__link">
5. Monitor Services
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#ancillary-edex-server-ingestdecode-edex-server" title="Ancillary EDEX Server (Ingest/Decode EDEX Server)" class="md-nav__link">
Ancillary EDEX Server (Ingest/Decode EDEX Server)
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#1-install_1" title="1. Install" class="md-nav__link">
1. Install
</a>
</li>
<li class="md-nav__item">
<a href="#2-edex-config" title="2. EDEX Config" class="md-nav__link">
2. EDEX Config
</a>
</li>
<li class="md-nav__item">
<a href="#3-modify-the-edexservicelist" title="3. Modify the edexServiceList" class="md-nav__link">
3. Modify the edexServiceList
</a>
</li>
<li class="md-nav__item">
<a href="#4-configure-your-ldm" title="4. Configure your LDM" class="md-nav__link">
4. Configure your LDM
</a>
</li>
<li class="md-nav__item">
<a href="#5-start-edex" title="5. Start EDEX" class="md-nav__link">
5. Start EDEX
</a>
</li>
<li class="md-nav__item">
<a href="#4-monitor-services" title="4. Monitor Services" class="md-nav__link">
4. Monitor Services
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#additional-notes" title="Additional Notes" class="md-nav__link">
Additional Notes
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content">
<article class="md-content__inner md-typeset">
<a href="https://github.com/Unidata/awips2/edit/unidata_20.3.2/docs/edex/distributed-computing.md" title="Edit this page" class="md-icon md-content__icon">&#xE3C9;</a>
<h1 id="distributed-edex">Distributed EDEX<a class="headerlink" href="#distributed-edex" title="Permanent link"></a></h1>
<p>AWIPS makes use of service-oriented architecture to request, process, and serve real-time meteorological data. While originally developed for use on internal NWS forecast office networks, where operational installations of AWIPS could consist of a dozen servers or more, the early NSF Unidata releases were stripped of operations-specific configurations and plugins, and released as a standalone server. This worked, since (at the time) a single EDEX instance with an attached SSD could handle most of NOAAport. However, with GOES-R(16) coming online in 2017, and more gridded forecast models being created at finer temporal and spatial resolutions, there is now a need to distribute the data decoding across multiple machines to handle this firehose of data.</p>
<hr />
<h2 id="nsf-unidatas-current-edex-server">NSF Unidata's Current EDEX Server<a class="headerlink" href="#nsf-unidatas-current-edex-server" title="Permanent link"></a></h2>
<p>Currently, we use a distributed architecture comprised of three machines: 1 <strong>main EDEX</strong> machine and 2 <strong>ancillary EDEX</strong> machines. The main EDEX machine decodes and processes the majority of the data, while serving and storing all of the data. Our two ancillary machines -- one for radar data and one for satellite data -- each decode and process a subset of the data and send it back to the main EDEX for storage and requesting.</p>
<p>The main EDEX is an instance of our <a href="#databaserequest-server">Database and Request Server</a> and more information on our <a href="#ancillary-edex-server-ingestdecode-edex-server">ancillary EDEX machines</a> is below as well.</p>
<hr />
<h2 id="example-installation">Example Installation<a class="headerlink" href="#example-installation" title="Permanent link"></a></h2>
<p>This walkthrough will install different EDEX components on two machines in the XSEDE Jetstream Cloud, the first is used to <strong>store and serve</strong> while the second is used to <strong>ingest and decode</strong> data.</p>
<p><img alt="" src="/images/awips2_distributed.png" /></p>
<hr />
<h3 id="databaserequest-server">Database/Request Server<a class="headerlink" href="#databaserequest-server" title="Permanent link"></a></h3>
<p>For this example, this server will be referred to by the IP address <strong>10.0.0.9</strong>.</p>
<h4 id="1-install">1. Install<a class="headerlink" href="#1-install" title="Permanent link"></a></h4>
<pre><code>wget https://downloads.unidata.ucar.edu/awips2/current/linux/awips_install.sh
chmod 755 awips_install.sh
sudo ./awips_install.sh --database
</code></pre>
<h4 id="2-iptables-config">2. IPtables Config<a class="headerlink" href="#2-iptables-config" title="Permanent link"></a></h4>
<p>It is required that ports 5432 and 5672 be open for the specific IP addresses of outside EDEX ingest servers. It is <em>not recommended</em> that you leave port 5432 open to all connections (since the default awips database password is known, and is not meant as a security measure). Further, it <em>is recommended</em> that you change the default postgres awips user password (which then requires a reconfiguration of every remote EDEX ingest server in order to connect to this database/request server).</p>
<pre><code>vi /etc/sysconfig/iptables
*filter
:INPUT DROP [0:0]
:FORWARD DROP [0:0]
:OUTPUT ACCEPT [0:0]
:EXTERNAL - [0:0]
:EDEX - [0:0]
-A INPUT -i lo -j ACCEPT
-A INPUT -p icmp --icmp-type any -j ACCEPT
-A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
-A INPUT -m state --state NEW -m tcp -p tcp --dport 22 -j ACCEPT
-A INPUT -m state --state NEW -m tcp -p tcp --dport 9581 -j ACCEPT
-A INPUT -m state --state NEW -m tcp -p tcp --dport 9582 -j ACCEPT
-A INPUT -s 10.0.0.7 -j EDEX
-A INPUT -j EXTERNAL
-A EXTERNAL -j REJECT
-A EDEX -m state --state NEW -p tcp --dport 5432 -j ACCEPT
-A EDEX -m state --state NEW -p tcp --dport 5672 -j ACCEPT
-A EDEX -j REJECT
COMMIT
</code></pre>
<p>Note the line <strong><code>-A INPUT -s 10.0.0.7 -j EDEX</code></strong> as well as the following <strong><code>-A EDEX ...</code></strong> rules for ports 5432 (PostgreSQL) and 5672 (PyPIES/HDF5). </p>
<div class="admonition note">
<p class="admonition-title">The two ports left open to all connections (9581,9582) in addition to default port 22 are for outside CAVE client connections</p>
</div>
<h4 id="3-database-config">3. Database Config<a class="headerlink" href="#3-database-config" title="Permanent link"></a></h4>
<p>In the file <code>/awips2/database/data/pg_hba.conf</code> you define remote connections for all postgres tables with as <code>&lt;IP address&gt;/32</code>, after the block of IPv4 local connections and generic <code>&lt;IP address/24&gt;</code> for hostnossl:</p>
<pre><code>vi /awips2/database/data/pg_hba.conf
# "local" is for Unix domain socket connections only
local all all trust
hostssl all all 162.0.0.0/8 cert clientcert=1
hostssl all all 127.0.0.1/32 cert clientcert=1
hostssl all all 10.0.0.7/32 cert clientcert=1
hostnossl postgres all 10.0.0.0/24 md5
hostnossl fxatext all 10.0.0.0/24 md5
hostnossl metadata all 10.0.0.0/24 md5
# IPv6 local connections:
hostssl all all ::1/128 cert clientcert=1
hostnossl all all ::1/128 md5
</code></pre>
<h4 id="4-start-edex">4. Start EDEX<a class="headerlink" href="#4-start-edex" title="Permanent link"></a></h4>
<pre><code>edex start database
</code></pre>
<p>This will start PostgreSQL, httpd-pypies, Qpid, and the EDEX Request JVM (and will not start the LDM or the EDEX Ingest and IngestGrib JVMs)</p>
<h4 id="5-monitor-services">5. Monitor Services<a class="headerlink" href="#5-monitor-services" title="Permanent link"></a></h4>
<p>The command <code>edex</code> will show which services are running, and for a Database/Request server, will not include the LDM, EDEXingest, or EDEXgrib:</p>
<pre><code>edex
[edex status]
postgres :: running :: pid 571
pypies :: running :: pid 639
qpid :: running :: pid 674
EDEXingest :: not running
EDEXgrib :: not running
EDEXrequest :: running :: pid 987 1029 23792
</code></pre>
<p>Since this Database/Request server is not running the main <em>edexIngest</em> JVM, we won't see anything from <code>edex log</code>, instead watch the Request Server with the command</p>
<pre><code>edex log request
</code></pre>
<div class="admonition warning">
<p class="admonition-title">Confirm that EDEX Request connects to PostgreSQL!</p>
<p>With the above <code>edex log request</code>, ensure that the log progresses <strong>past this point</strong>:</p>
<pre><code>Spring-enabled Plugins:
-----------------------
acars-common, acars-common-dataaccess, acarssounding-common, activetable-common,
activetable-request, airep-common, airep-common-dataaccess, airmet-common,
atcf-common, atcf-request, auth-request, awipstools-request, aww-common...
JAXB context for PersistencePathKeySet inited in: 5ms
INFO 20:21:09,134 5584 [EDEXMain] Reflections: Reflections took 436 ms to scan 258 urls, producing 31 keys and 3637 values
Found 499 db classes in 720 ms
</code></pre>
<p>If the log stops at the <strong>Found db classes...</strong> line, that means EDEX is not connecting to PostgreSQL - double-check <code>DB_ADDR</code> in <code>/awips2/edex/bin/setup.env</code></p>
</div>
<hr />
<h3 id="ancillary-edex-server-ingestdecode-edex-server">Ancillary EDEX Server (Ingest/Decode EDEX Server)<a class="headerlink" href="#ancillary-edex-server-ingestdecode-edex-server" title="Permanent link"></a></h3>
<p>For this example, this server will be referred to by the IP address <strong>10.0.0.7</strong>.
The <strong>Main EDEX</strong> server will be referred to by the IP address <strong>10.0.0.9</strong>.</p>
<h4 id="1-install_1">1. Install<a class="headerlink" href="#1-install_1" title="Permanent link"></a></h4>
<pre><code>wget https://downloads.unidata.ucar.edu/awips2/current/linux/awips_install.sh
chmod 755 awips_install.sh
sudo ./awips_install.sh --ingest
</code></pre>
<h4 id="2-edex-config">2. EDEX Config<a class="headerlink" href="#2-edex-config" title="Permanent link"></a></h4>
<p><code>vi /awips2/edex/bin/setup.env</code></p>
<p>Here you should redefine <code>DB_ADDR</code> and <code>PYPIES_SERVER</code> to point to the <strong>Main</strong> or <strong>Database/Request</strong> server (10.0.0.9) and the <code>EXT_ADDR</code> to point to the current <strong>Ingest</strong> server (10.0.0.7)</p>
<pre><code>export EXT_ADDR=10.0.0.7
# postgres connection
export DB_ADDR=10.0.0.9
export DB_PORT=5432
# pypies hdf5 connection
export PYPIES_SERVER=http://10.0.0.9:9582
# qpid connection
export BROKER_ADDR=${EXT_ADDR}
</code></pre>
<p>Notice that <code>EXT_ADDR</code> and <code>BROKER_ADDR</code> (qpid) should remain defined as the <em>localhost</em> IP address (10.0.0.7)</p>
<h4 id="3-modify-the-edexservicelist">3. Modify the edexServiceList<a class="headerlink" href="#3-modify-the-edexservicelist" title="Permanent link"></a></h4>
<p>Most likely if you are running a distributed EDEX setup, you are only processing a subset of data. You can change your edexServiceList to only run the processes you need. You will need to update the <code>/etc/init.d/edexServiceList</code> file. For example replace the services with the associated right column based on the data you're processing:</p>
<pre><code>export SERVICES=('')
</code></pre>
<table>
<thead>
<tr>
<th>Data Processing:</th>
<th>edexServiceList</th>
</tr>
</thead>
<tbody>
<tr>
<td>radar</td>
<td>ingestRadar</td>
</tr>
<tr>
<td>satellite</td>
<td>ingestGoesR</td>
</tr>
<tr>
<td>model</td>
<td>ingestGrids, ingestGrib</td>
</tr>
</tbody>
</table>
<h4 id="4-configure-your-ldm">4. Configure your LDM<a class="headerlink" href="#4-configure-your-ldm" title="Permanent link"></a></h4>
<p>You'll want to modify your pqact.conf file to store only the data you want processed. There are example files in <code>/awips2/ldm/etc</code> that you can copy over to the main pqact.conf file. For example if you are wanting to process goesr data only, you can do the following steps:</p>
<pre><code>cd /awips2/ldm/etc
mv pqact.conf pqact.conf.orig
cp pqact.goesr pqact.conf
</code></pre>
<p>You will also want to edit the <code>pqact.conf</code> file on your <strong>Main EDEX</strong> and comment out any entries you're processing on this EDEX server. </p>
<h4 id="5-start-edex">5. Start EDEX<a class="headerlink" href="#5-start-edex" title="Permanent link"></a></h4>
<pre><code>edex start
</code></pre>
<p>This will start LDM, Qpid and the specified EDEX Ingest JVMs (and not start PostgreSQL, httpd-pypies, or the EDEX Request JVM)</p>
<h4 id="4-monitor-services">4. Monitor Services<a class="headerlink" href="#4-monitor-services" title="Permanent link"></a></h4>
<p>Watch the edex JVM log with the command</p>
<pre><code>edex log
</code></pre>
<div class="admonition warning">
<p class="admonition-title">Confirm that EDEX connects to PostgreSQL!</p>
<p>With the above <code>edex log</code>, ensure that the log progresses <strong>past this point</strong>:</p>
<pre><code>Spring-enabled Plugins:
-----------------------
acars-common, acars-common-dataaccess, acarssounding-common, activetable-common,
activetable-ingest, airep-common, airep-common-dataaccess, airmet-common,
atcf-common, atcf-ingest, aww-common...
JAXB context for PersistencePathKeySet inited in: 5ms
INFO 20:21:09,134 5584 [EDEXMain] Reflections: Reflections took 436 ms to scan 258 urls, producing 31 keys and 3637 values
Found 499 db classes in 720 ms
</code></pre>
<p>If the log stops at the <strong>Found db classes...</strong> line, that means EDEX is not connecting to the <em>remote PostgreSQL instance</em> - double-check <code>DB_ADDR</code> in <code>/awips2/edex/bin/setup.env</code></p>
<p>You can <strong>manually check remote PostgreSQL connectivity</strong> on any EDEX Ingest server from the command line:</p>
<pre><code>su - awips
psql -U awips -h &lt;remote IP address&gt; -p 5432 metadata
</code></pre>
<p>Where the default passwd is <em>awips</em> and is defined in files in <code>/awips2/edex/conf/db/hibernateConfig/</code></p>
</div>
<hr />
<h2 id="additional-notes">Additional Notes<a class="headerlink" href="#additional-notes" title="Permanent link"></a></h2>
<ul>
<li>Be mindful of what IP address and hostnames are used in <code>/awips2/edex/bin/setup.env</code> and <code>/awips2/database/data/pg_hba.conf</code>, and that they are resolvable from the command line. Consult or edit <code>/etc/hosts</code> as needed.</li>
<li>
<p>You can install multiple <code>awips2-ingest</code> servers, each decoding a different dataset or feed, all pointing to the same Database/Request server (<code>DB_ADDR</code> and <code>PYPIES_SERVER</code> in <code>/awips2/edex/bin/setup.env</code>):</p>
</li>
<li>
<p>Every EDEX Ingest IP address must be allowed in both <strong>iptables</strong> and <strong>pg_hba.conf</strong> as <a href="#2-iptables-config">shown above</a>.</p>
</li>
</ul>
</article>
</div>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-nav">
<nav class="md-footer-nav__inner md-grid">
<a href="edex/settings/" title="EDEX Settings" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
<div class="md-flex__cell md-flex__cell--shrink">
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
</div>
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
<span class="md-flex__ellipsis">
<span class="md-footer-nav__direction">
Previous
</span>
EDEX Settings
</span>
</div>
</a>
<a href="edex/edex-ingest-docker-container/" title="Docker EDEX" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
<span class="md-flex__ellipsis">
<span class="md-footer-nav__direction">
Next
</span>
Docker EDEX
</span>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
</div>
</a>
</nav>
</div>
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-footer-copyright">
<div class="md-footer-copyright__highlight">
Copyright &copy; 2023 NSF Unidata Program Center.
</div>
powered by
<a href="http://www.mkdocs.org" title="MkDocs">MkDocs</a>
and
<a href="http://squidfunk.github.io/mkdocs-material/" title="Material for MkDocs">
Material for MkDocs</a>
</div>
<div class="md-footer-social">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
<a href="https://www.unidata.ucar.edu/software/awips2" class="md-footer-social__link fa fa-globe"></a>
<a href="https://github.com/Unidata/awips2" class="md-footer-social__link fa fa-github-alt"></a>
<a href="https://twitter.com/Unidata" class="md-footer-social__link fa fa-twitter"></a>
<a href="https://facebook.com/Unidata" class="md-footer-social__link fa fa-facebook"></a>
</div>
</div>
</div>
</footer>
</div>
<script src="../../assets/javascripts/application-eaeb7b21e3.js"></script>
<script>app.initialize({url:{base:"../.."}})</script>
</body>
</html>
Reference in a new issue View git blame Copy permalink