xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
awips2/edex/distributed-computing/index.html
srcarter3 910ddff923 deploy: 9df118bd43
2021-08-18 19:50:54 +00:00

1286 lines
No EOL
36 KiB
HTML
Raw Blame History

<!DOCTYPE html>
<html class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="Documentation for all things AWIPS.">
<link rel="canonical" href="http://unidata.github.io/awips2/edex/distributed-computing/">
<meta name="author" content="support-awips@unidata.ucar.edu">
<link rel="shortcut icon" href="../../images/favicon.ico">
<meta name="generator" content="mkdocs-0.17.5, mkdocs-unidata-1.5.4">
<title>Distributed EDEX - Unidata AWIPS User Manual</title>
<script src="../../assets/javascripts/modernizr-56ade86843.js"></script>
<link rel="stylesheet" href="../../assets/stylesheets/application-b1a1975878.css">
<link rel="stylesheet" href="../../assets/stylesheets/application-f78e5cb881.palette.css">
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono">
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
<link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
<link rel="stylesheet" href="../../css/extra.css">
</head>
<body data-md-color-primary="blue-grey" data-md-color-accent="light-blue">
<svg class="md-svg">
<defs>
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
</defs>
</svg>
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="drawer">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="search">
<label class="md-overlay" data-md-component="overlay" for="drawer"></label>
<header class="md-header" data-md-component="header">
<nav class="md-header-nav md-grid">
<a href="../.."><div class="background-logo" style="background-image: url(../../images/cave.png);"></div>
<div class="md-flex">
<div class="md-flex__cell md-flex__cell--shrink">
<a href="../.." title="Unidata AWIPS User Manual" class="md-icon md-icon--home md-header-nav__button">
</a>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<label class="md-icon md-icon--menu md-header-nav__button" for="drawer"></label>
</div>
<div class="md-flex__cell md-flex__cell--stretch">
<span class="md-flex__ellipsis md-header-nav__title">
<span class="md-header-nav__parent">
EDEX User Manual
</span>
Distributed EDEX
</span>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<label class="md-icon md-icon--search md-header-nav__button" for="search"></label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="search"></label>
<div class="md-search__inner">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" required placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query">
<label class="md-icon md-search__icon" for="search"></label>
<button type="reset" class="md-icon md-search__icon" data-md-component="reset">&#xE5CD;</button>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" data-md-scrollfix>
<div class="md-search-result" data-md-component="result" data-md-lang-search="" data-md-lang-tokenizer="[\s\-]+">
<div class="md-search-result__meta" data-md-lang-result-none="No matching documents" data-md-lang-result-one="1 matching document" data-md-lang-result-other="# matching documents">
Type to start searching
</div>
<ol class="md-search-result__list"></ol>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<div class="md-header-nav__source">
<a href="https://github.com/Unidata/awips2/" title="Go to repository" class="md-source" data-md-source="github">
<div class="md-source__icon">
<svg viewBox="0 0 24 24" width="24" height="24">
<use xlink:href="#github" width="24" height="24"></use>
</svg>
</div>
<div class="md-source__repository">
GitHub
</div>
</a>
</div>
</div>
</div>
</nav>
</header>
<div class="md-container">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" data-md-level="0">
<label class="md-nav__title md-nav__title--site" for="drawer">
<div class="md-nav__button md-logo">
<i class="md-icon md-icon--home"></i>
</div>
Unidata AWIPS User Manual
</label>
<div class="md-nav__source">
<a href="https://github.com/Unidata/awips2/" title="Go to repository" class="md-source" data-md-source="github">
<div class="md-source__icon">
<svg viewBox="0 0 24 24" width="24" height="24">
<use xlink:href="#github" width="24" height="24"></use>
</svg>
</div>
<div class="md-source__repository">
GitHub
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../.." title="Home" class="md-nav__link">
Home
</a>
</li>
<li class="md-nav__item">
<a href="../../install/install-cave/" title="Install CAVE" class="md-nav__link">
Install CAVE
</a>
</li>
<li class="md-nav__item">
<a href="../../install/install-edex/" title="Install EDEX" class="md-nav__link">
Install EDEX
</a>
</li>
<li class="md-nav__item">
<a href="../../install/common-problems/" title="Common Problems" class="md-nav__link">
Common Problems
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-5" type="checkbox" id="nav-5">
<label class="md-nav__link" for="nav-5">
CAVE User Manual
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
<label class="md-nav__title" for="nav-5">
CAVE User Manual
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../cave/d2d-perspective/" title="D2D Perspective" class="md-nav__link">
D2D Perspective
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/maps-views-projections/" title="Maps, Views, Projections" class="md-nav__link">
Maps, Views, Projections
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/bundles-and-procedures/" title="Displays and Procedures" class="md-nav__link">
Displays and Procedures
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/localization-perspective/" title="Localization Perspective" class="md-nav__link">
Localization Perspective
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/nsharp/" title="NSHARP" class="md-nav__link">
NSHARP
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/warngen/" title="WarnGen Walkthrough" class="md-nav__link">
WarnGen Walkthrough
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/goes-16-17-satellite/" title="GOES 16/17" class="md-nav__link">
GOES 16/17
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-gis-shapefiles/" title="GIS and Shapefiles" class="md-nav__link">
GIS and Shapefiles
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-gridded-models/" title="Volume Browser" class="md-nav__link">
Volume Browser
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-tools/" title="Display Tools" class="md-nav__link">
Display Tools
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-radar-tools/" title="Radar Tools" class="md-nav__link">
Radar Tools
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-edit-menus/" title="Editing Menus" class="md-nav__link">
Editing Menus
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/cave-localization/" title="Change Localization" class="md-nav__link">
Change Localization
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/import-export/" title="Import/Export" class="md-nav__link">
Import/Export
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-6" type="checkbox" id="nav-6" checked>
<label class="md-nav__link" for="nav-6">
EDEX User Manual
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
<label class="md-nav__title" for="nav-6">
EDEX User Manual
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../install/start-edex/" title="EDEX Basic Commands" class="md-nav__link">
EDEX Basic Commands
</a>
</li>
<li class="md-nav__item">
<a href="../settings/" title="EDEX Settings" class="md-nav__link">
EDEX Settings
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="toc">
<label class="md-nav__link md-nav__link--active" for="toc">
Distributed EDEX
</label>
<a href="./" title="Distributed EDEX" class="md-nav__link md-nav__link--active">
Distributed EDEX
</a>
<nav class="md-nav md-nav--secondary">
<label class="md-nav__title" for="toc">Table of contents</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#unidatas-current-edex-server" title="Unidata's Current EDEX Server" class="md-nav__link">
Unidata's Current EDEX Server
</a>
</li>
<li class="md-nav__item">
<a href="#example-installation" title="Example Installation" class="md-nav__link">
Example Installation
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#databaserequest-server" title="Database/Request Server" class="md-nav__link">
Database/Request Server
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#1-install" title="1. Install" class="md-nav__link">
1. Install
</a>
</li>
<li class="md-nav__item">
<a href="#2-iptables-config" title="2. IPtables Config" class="md-nav__link">
2. IPtables Config
</a>
</li>
<li class="md-nav__item">
<a href="#3-database-config" title="3. Database Config" class="md-nav__link">
3. Database Config
</a>
</li>
<li class="md-nav__item">
<a href="#4-start-edex" title="4. Start EDEX" class="md-nav__link">
4. Start EDEX
</a>
</li>
<li class="md-nav__item">
<a href="#5-monitor-services" title="5. Monitor Services" class="md-nav__link">
5. Monitor Services
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#ingestdecode-server" title="Ingest/Decode Server" class="md-nav__link">
Ingest/Decode Server
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#1-install_1" title="1. Install" class="md-nav__link">
1. Install
</a>
</li>
<li class="md-nav__item">
<a href="#2-edex-config" title="2. EDEX Config" class="md-nav__link">
2. EDEX Config
</a>
</li>
<li class="md-nav__item">
<a href="#3-start-edex" title="3. Start EDEX" class="md-nav__link">
3. Start EDEX
</a>
</li>
<li class="md-nav__item">
<a href="#4-monitor-services" title="4. Monitor Services" class="md-nav__link">
4. Monitor Services
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#additional-notes" title="Additional Notes" class="md-nav__link">
Additional Notes
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../edex-ingest-docker-container/" title="Docker EDEX" class="md-nav__link">
Docker EDEX
</a>
</li>
<li class="md-nav__item">
<a href="../ldm/" title="LDM Feeds" class="md-nav__link">
LDM Feeds
</a>
</li>
<li class="md-nav__item">
<a href="../data-distribution-files/" title="Data Distribution Files" class="md-nav__link">
Data Distribution Files
</a>
</li>
<li class="md-nav__item">
<a href="../new-grid/" title="Ingest a New Grid" class="md-nav__link">
Ingest a New Grid
</a>
</li>
<li class="md-nav__item">
<a href="../data-purge/" title="Purging and Retention" class="md-nav__link">
Purging and Retention
</a>
</li>
<li class="md-nav__item">
<a href="../edex-users/" title="Monitor Users" class="md-nav__link">
Monitor Users
</a>
</li>
<li class="md-nav__item">
<a href="../data-plugins/" title="Data Plugins" class="md-nav__link">
Data Plugins
</a>
</li>
<li class="md-nav__item">
<a href="../case-studies/" title="Archive Case Studies" class="md-nav__link">
Archive Case Studies
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../../python/overview/" title="Python-AWIPS" class="md-nav__link">
Python-AWIPS
</a>
</li>
<li class="md-nav__item">
<a href="../../dev/awips-development-environment/" title="Development" class="md-nav__link">
Development
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-9" type="checkbox" id="nav-9">
<label class="md-nav__link" for="nav-9">
Appendix
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
<label class="md-nav__title" for="nav-9">
Appendix
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../appendix/appendix-grid-parameters/" title="AWIPS Grid Parameters" class="md-nav__link">
AWIPS Grid Parameters
</a>
</li>
<li class="md-nav__item">
<a href="../../python/maps-database/" title="Maps Database" class="md-nav__link">
Maps Database
</a>
</li>
<li class="md-nav__item">
<a href="../../appendix/appendix-acronyms/" title="Acronyms and Abbreviations" class="md-nav__link">
Acronyms and Abbreviations
</a>
</li>
<li class="md-nav__item">
<a href="../../appendix/appendix-wsr88d/" title="WSR-88D Product Table" class="md-nav__link">
WSR-88D Product Table
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/cave-keyboard-shortcuts/" title="Keyboard Shortcuts" class="md-nav__link">
Keyboard Shortcuts
</a>
</li>
<li class="md-nav__item">
<a href="../../raytheon/cave_d2d/" title="Raytheon: CAVE User's Manual" class="md-nav__link">
Raytheon: CAVE User's Manual
</a>
</li>
<li class="md-nav__item">
<a href="../../raytheon/smm/" title="Raytheon: AWIPS System Manager's Manual" class="md-nav__link">
Raytheon: AWIPS System Manager's Manual
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary">
<label class="md-nav__title" for="toc">Table of contents</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#unidatas-current-edex-server" title="Unidata's Current EDEX Server" class="md-nav__link">
Unidata's Current EDEX Server
</a>
</li>
<li class="md-nav__item">
<a href="#example-installation" title="Example Installation" class="md-nav__link">
Example Installation
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#databaserequest-server" title="Database/Request Server" class="md-nav__link">
Database/Request Server
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#1-install" title="1. Install" class="md-nav__link">
1. Install
</a>
</li>
<li class="md-nav__item">
<a href="#2-iptables-config" title="2. IPtables Config" class="md-nav__link">
2. IPtables Config
</a>
</li>
<li class="md-nav__item">
<a href="#3-database-config" title="3. Database Config" class="md-nav__link">
3. Database Config
</a>
</li>
<li class="md-nav__item">
<a href="#4-start-edex" title="4. Start EDEX" class="md-nav__link">
4. Start EDEX
</a>
</li>
<li class="md-nav__item">
<a href="#5-monitor-services" title="5. Monitor Services" class="md-nav__link">
5. Monitor Services
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#ingestdecode-server" title="Ingest/Decode Server" class="md-nav__link">
Ingest/Decode Server
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#1-install_1" title="1. Install" class="md-nav__link">
1. Install
</a>
</li>
<li class="md-nav__item">
<a href="#2-edex-config" title="2. EDEX Config" class="md-nav__link">
2. EDEX Config
</a>
</li>
<li class="md-nav__item">
<a href="#3-start-edex" title="3. Start EDEX" class="md-nav__link">
3. Start EDEX
</a>
</li>
<li class="md-nav__item">
<a href="#4-monitor-services" title="4. Monitor Services" class="md-nav__link">
4. Monitor Services
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#additional-notes" title="Additional Notes" class="md-nav__link">
Additional Notes
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content">
<article class="md-content__inner md-typeset">
<a href="https://github.com/Unidata/awips2/edit/unidata_18.2.1/docs/edex/distributed-computing.md" title="Edit this page" class="md-icon md-content__icon">edit</a>
<h1 id="distributed-edex">Distributed EDEX<a class="headerlink" href="#distributed-edex" title="Permanent link"></a></h1>
<p>AWIPS makes use of service-oriented architecture to request, process, and serve real-time meteorological data. While originally developed for use on internal NWS forecast office networks, where operational installations of AWIPS could consist of a dozen servers or more, the early Unidata releases were stripped of operations-specific configurations and plugins, and released as a standalone server. This worked, since (at the time) a single EDEX instance with an attached SSD could handle most of NOAAport. However, with GOES-R(16) coming online in 2017, and more gridded forecast models being created at finer temporal and spatial resolutions, there is now a need to distribute the data decoding across multiple machines to handle this firehose of data.</p>
<hr />
<h2 id="unidatas-current-edex-server">Unidata's Current EDEX Server<a class="headerlink" href="#unidatas-current-edex-server" title="Permanent link"></a></h2>
<p>Currently, with our specific EDEX server we use a Database/Request instance that also decodes and ingests a good portion of the data. It handles all data requests from CAVE users, as well as the majority of the decoding and ingesting for data feeds coming down on the LDM. The <strong>radar</strong> data has been specifically exluded (from the decoding and ingest) and it has its own <a href="#ingestdecode-server"><strong>Ingest/Decode Server</strong></a> which is explained in more detail below.</p>
<p>For our EDEX we have designated an instance of the ingest/decoding server to be dedicated to handling the radar data. Our <em>Radar-EDEX</em> recieves and decodes all radar down from the LDM and then stores it back on our main <a href="#databaserequest-server"><strong>Database/Request EDEX</strong></a> in the form of HDF5 data files and PostgreSQL metadata.</p>
<hr />
<h2 id="example-installation">Example Installation<a class="headerlink" href="#example-installation" title="Permanent link"></a></h2>
<p>This walkthrough will install different EDEX components on two machines in the XSEDE Jetstream Cloud, the first is used to <strong>store and serve</strong> while the second is used to <strong>ingest and decode</strong> data.</p>
<p><img alt="" src="../../images/awips2_distributed.png" /></p>
<hr />
<h3 id="databaserequest-server">Database/Request Server<a class="headerlink" href="#databaserequest-server" title="Permanent link"></a></h3>
<p>For this example, this server will be referred to by the IP address <strong>10.0.0.9</strong>.</p>
<h4 id="1-install">1. Install<a class="headerlink" href="#1-install" title="Permanent link"></a></h4>
<pre><code>groupadd fxalpha &amp;&amp; useradd -G fxalpha awips
mkdir /awips2
wget -O /etc/yum.repos.d/awips2.repo https://downloads.unidata.ucar.edu/awips2/current/linux/awips2.repo
yum clean all
yum groupinstall awips2-database
</code></pre>
<h4 id="2-iptables-config">2. IPtables Config<a class="headerlink" href="#2-iptables-config" title="Permanent link"></a></h4>
<p>It is required that ports 5432 and 5672 be open for the specific IP addresses of outside EDEX ingest servers. It is <em>not recommended</em> that you leave port 5432 open to all connections (since the default awips database password is known, and is not meant as a security measure). Further, it <em>is recommended</em> that you change the default postgres awips user password (which then requires a reconfiguration of every remote EDEX ingest server in order to connect to this database/request server).</p>
<pre><code>vi /etc/sysconfig/iptables
*filter
:INPUT DROP [0:0]
:FORWARD DROP [0:0]
:OUTPUT ACCEPT [0:0]
:EXTERNAL - [0:0]
:EDEX - [0:0]
-A INPUT -i lo -j ACCEPT
-A INPUT -p icmp --icmp-type any -j ACCEPT
-A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
-A INPUT -m state --state NEW -m tcp -p tcp --dport 22 -j ACCEPT
-A INPUT -m state --state NEW -m tcp -p tcp --dport 9581 -j ACCEPT
-A INPUT -m state --state NEW -m tcp -p tcp --dport 9582 -j ACCEPT
-A INPUT -s 10.0.0.7 -j EDEX
-A INPUT -j EXTERNAL
-A EXTERNAL -j REJECT
-A EDEX -m state --state NEW -p tcp --dport 5432 -j ACCEPT
-A EDEX -m state --state NEW -p tcp --dport 5672 -j ACCEPT
-A EDEX -j REJECT
COMMIT
</code></pre>
<p>Note the line <strong><code>-A INPUT -s 10.0.0.7 -j EDEX</code></strong> as well as the following <strong><code>-A EDEX ...</code></strong> rules for ports 5432 (PostgreSQL) and 5672 (PyPIES/HDF5). </p>
<div class="admonition note">
<p class="admonition-title">The two ports left open to all connections (9581,9582) in addition to default port 22 are for outside CAVE client connections</p>
</div>
<h4 id="3-database-config">3. Database Config<a class="headerlink" href="#3-database-config" title="Permanent link"></a></h4>
<p>In the file <code>/awips2/database/data/pg_hba.conf</code> you define remote connections for all postgres tables with as <code>&lt;IP address&gt;/32</code>, after the block of IPv4 local connections:</p>
<pre><code>vi /awips2/database/data/pg_hba.conf
# "local" is for Unix domain socket connections only
local all all trust
hostssl all all 10.0.0.7/32 cert clientcert=1
hostssl all all 162.0.0.0/8 cert clientcert=1
hostssl all all 127.0.0.1/32 cert clientcert=1
# IPv6 local connections:
hostssl all all ::1/128 cert clientcert=1
hostnossl all all ::1/128 md5
</code></pre>
<h4 id="4-start-edex">4. Start EDEX<a class="headerlink" href="#4-start-edex" title="Permanent link"></a></h4>
<pre><code>edex start database
</code></pre>
<p>This will start PostgreSQL, httpd-pypies, Qpid, and the EDEX Request JVM (and will not start the LDM or the EDEX Ingest and IngestGrib JVMs)</p>
<h4 id="5-monitor-services">5. Monitor Services<a class="headerlink" href="#5-monitor-services" title="Permanent link"></a></h4>
<p>The command <code>edex</code> will show which services are running, and for a Database/Request server, will not include the LDM, EDEXingest, or EDEXgrib:</p>
<pre><code>edex
[edex status]
postgres :: running :: pid 571
pypies :: running :: pid 639
qpid :: running :: pid 674
EDEXingest :: not running
EDEXgrib :: not running
EDEXrequest :: running :: pid 987 1029 23792
</code></pre>
<p>Since this Database/Request server is not running the main <em>edexIngest</em> JVM, we won't see anything from <code>edex log</code>, instead watch the Request Server with the command</p>
<pre><code>edex log request
</code></pre>
<div class="admonition warning">
<p class="admonition-title">Confirm that EDEX Request connects to PostgreSQL!</p>
<p>With the above <code>edex log request</code>, ensure that the log progresses <strong>past this point</strong>:</p>
<pre><code>Spring-enabled Plugins:
-----------------------
acars-common, acars-common-dataaccess, acarssounding-common, activetable-common,
activetable-request, airep-common, airep-common-dataaccess, airmet-common,
atcf-common, atcf-request, auth-request, awipstools-request, aww-common...
JAXB context for PersistencePathKeySet inited in: 5ms
INFO 20:21:09,134 5584 [EDEXMain] Reflections: Reflections took 436 ms to scan 258 urls, producing 31 keys and 3637 values
Found 499 db classes in 720 ms
</code></pre>
<p>If the log stops at the <strong>Found db classes...</strong> line, that means EDEX is not connecting to PostgreSQL - double-check <code>DB_ADDR</code> in <code>/awips2/edex/bin/setup.env</code></p>
</div>
<hr />
<h3 id="ingestdecode-server">Ingest/Decode Server<a class="headerlink" href="#ingestdecode-server" title="Permanent link"></a></h3>
<p>For this example, this server will be referred to by the IP address <strong>10.0.0.7</strong>.</p>
<h4 id="1-install_1">1. Install<a class="headerlink" href="#1-install_1" title="Permanent link"></a></h4>
<pre><code>groupadd fxalpha &amp;&amp; useradd -G fxalpha awips
wget -O /etc/yum.repos.d/awips2.repo https://downloads.unidata.ucar.edu/awips2/current/linux/awips2.repo
yum clean all
yum groupinstall awips2-ingest
</code></pre>
<h4 id="2-edex-config">2. EDEX Config<a class="headerlink" href="#2-edex-config" title="Permanent link"></a></h4>
<p><code>vi /awips2/edex/bin/setup.env</code></p>
<p>Here you should redefine <code>DB_ADDR</code> and <code>PYPIES_SERVER</code> to point to the <strong>Database/Request</strong> server (10.0.0.9)</p>
<pre><code>export EDEX_SERVER=10.0.0.7
# postgres connection
export DB_ADDR=10.0.0.9
export DB_PORT=5432
# pypies hdf5 connection
export PYPIES_SERVER=http://10.0.0.9:9582
# qpid connection
export BROKER_ADDR=${EDEX_SERVER}
</code></pre>
<p>Notice that <code>EDEX_SERVER</code> and <code>BROKER_ADDR</code> (qpid) should remain defined as the <em>localhost</em> IP address (10.0.0.7)</p>
<h4 id="3-start-edex">3. Start EDEX<a class="headerlink" href="#3-start-edex" title="Permanent link"></a></h4>
<pre><code>edex start ingest
</code></pre>
<p>This will start Qpid and the EDEX Ingest and IngestGrib JVMs (and not start PostgreSQL, httpd-pypies, or the EDEX Request JVM)</p>
<h4 id="4-monitor-services">4. Monitor Services<a class="headerlink" href="#4-monitor-services" title="Permanent link"></a></h4>
<p>Watch the edex JVM log with the command</p>
<pre><code>edex log
</code></pre>
<div class="admonition warning">
<p class="admonition-title">Confirm that EDEX connects to PostgreSQL!</p>
<p>With the above <code>edex log</code>, ensure that the log progresses <strong>past this point</strong>:</p>
<pre><code>Spring-enabled Plugins:
-----------------------
acars-common, acars-common-dataaccess, acarssounding-common, activetable-common,
activetable-ingest, airep-common, airep-common-dataaccess, airmet-common,
atcf-common, atcf-ingest, aww-common...
JAXB context for PersistencePathKeySet inited in: 5ms
INFO 20:21:09,134 5584 [EDEXMain] Reflections: Reflections took 436 ms to scan 258 urls, producing 31 keys and 3637 values
Found 499 db classes in 720 ms
</code></pre>
<p>If the log stops at the <strong>Found db classes...</strong> line, that means EDEX is not connecting to the <em>remote PostgreSQL instance</em> - double-check <code>DB_ADDR</code> in <code>/awips2/edex/bin/setup.env</code></p>
<p>You can <strong>manually check remote PostgreSQL connectivity</strong> on any EDEX Ingest server from the command line:</p>
<pre><code>su - awips
psql -U awips -h &lt;remote IP address&gt; -p 5432 metadata
</code></pre>
<p>Where the default passwd is <em>awips</em> and is defined in files in <code>/awips2/edex/conf/db/hibernateConfig/</code></p>
</div>
<hr />
<h2 id="additional-notes">Additional Notes<a class="headerlink" href="#additional-notes" title="Permanent link"></a></h2>
<ul>
<li>Be mindful of what IP address and hostnames are used in <code>/awips2/edex/bin/setup.env</code> and <code>/awips2/database/data/pg_hba.conf</code>, and that they are resolvable from the command line. Consult or edit <code>/etc/hosts</code> as needed.</li>
<li>
<p>You can install multiple <code>awips2-ingest</code> servers, each decoding a different dataset or feed, all pointing to the same Database/Request server (<code>DB_ADDR</code> and <code>PYPIES_SERVER</code> in <code>/awips2/edex/bin/setup.env</code>):</p>
</li>
<li>
<p>Every EDEX Ingest IP address must be allowed in both <strong>iptables</strong> and <strong>pg_hba.conf</strong> as <a href="#2-iptables-config">shown above</a>.</p>
</li>
</ul>
</article>
</div>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-nav">
<nav class="md-footer-nav__inner md-grid">
<a href="../settings/" title="EDEX Settings" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
<div class="md-flex__cell md-flex__cell--shrink">
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
</div>
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
<span class="md-flex__ellipsis">
<span class="md-footer-nav__direction">
Previous
</span>
EDEX Settings
</span>
</div>
</a>
<a href="../edex-ingest-docker-container/" title="Docker EDEX" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
<span class="md-flex__ellipsis">
<span class="md-footer-nav__direction">
Next
</span>
Docker EDEX
</span>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
</div>
</a>
</nav>
</div>
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-footer-copyright">
<div class="md-footer-copyright__highlight">
Copyright &copy; 2020 Unidata Program Center.
</div>
powered by
<a href="http://www.mkdocs.org" title="MkDocs">MkDocs</a>
and
<a href="http://squidfunk.github.io/mkdocs-material/" title="Material for MkDocs">
Material for MkDocs</a>
</div>
<div class="md-footer-social">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
<a href="https://www.unidata.ucar.edu/software/awips2" class="md-footer-social__link fa fa-globe"></a>
<a href="https://github.com/Unidata/awips2" class="md-footer-social__link fa fa-github-alt"></a>
<a href="https://twitter.com/Unidata" class="md-footer-social__link fa fa-twitter"></a>
<a href="https://facebook.com/Unidata" class="md-footer-social__link fa fa-facebook"></a>
</div>
</div>
</div>
</footer>
</div>
<script src="../../assets/javascripts/application-0b7df094bf.js"></script>
<script>app.initialize({url:{base:"../.."}})</script>
<script src="../../assets/javascripts/application-0b7df094bf.js"></script>
</body>
</html>
Reference in a new issue View git blame Copy permalink