xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
awips2/edex/data-purge/index.html
tiffanycmeyer13 7382729ba4 deploy: fbd06fbfbe
2021-10-20 13:26:34 +00:00

1232 lines
No EOL
35 KiB
HTML
Raw Blame History

<!DOCTYPE html>
<html class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="Documentation for all things AWIPS.">
<link rel="canonical" href="http://unidata.github.io/awips2/edex/data-purge/">
<meta name="author" content="support-awips@unidata.ucar.edu">
<link rel="shortcut icon" href="../../images/favicon.ico">
<meta name="generator" content="mkdocs-0.17.5, mkdocs-unidata-1.5.4">
<title>Purging and Retention - Unidata AWIPS User Manual</title>
<script src="../../assets/javascripts/modernizr-56ade86843.js"></script>
<link rel="stylesheet" href="../../assets/stylesheets/application-b1a1975878.css">
<link rel="stylesheet" href="../../assets/stylesheets/application-f78e5cb881.palette.css">
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono">
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
<link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
<link rel="stylesheet" href="../../css/extra.css">
</head>
<body data-md-color-primary="blue-grey" data-md-color-accent="light-blue">
<svg class="md-svg">
<defs>
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
</defs>
</svg>
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="drawer">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="search">
<label class="md-overlay" data-md-component="overlay" for="drawer"></label>
<header class="md-header" data-md-component="header">
<nav class="md-header-nav md-grid">
<a href="../.."><div class="background-logo" style="background-image: url(../../images/cave.png);"></div>
<div class="md-flex">
<div class="md-flex__cell md-flex__cell--shrink">
<a href="../.." title="Unidata AWIPS User Manual" class="md-icon md-icon--home md-header-nav__button">
</a>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<label class="md-icon md-icon--menu md-header-nav__button" for="drawer"></label>
</div>
<div class="md-flex__cell md-flex__cell--stretch">
<span class="md-flex__ellipsis md-header-nav__title">
<span class="md-header-nav__parent">
EDEX User Manual
</span>
Purging and Retention
</span>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<label class="md-icon md-icon--search md-header-nav__button" for="search"></label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="search"></label>
<div class="md-search__inner">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" required placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query">
<label class="md-icon md-search__icon" for="search"></label>
<button type="reset" class="md-icon md-search__icon" data-md-component="reset">&#xE5CD;</button>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" data-md-scrollfix>
<div class="md-search-result" data-md-component="result" data-md-lang-search="" data-md-lang-tokenizer="[\s\-]+">
<div class="md-search-result__meta" data-md-lang-result-none="No matching documents" data-md-lang-result-one="1 matching document" data-md-lang-result-other="# matching documents">
Type to start searching
</div>
<ol class="md-search-result__list"></ol>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<div class="md-header-nav__source">
<a href="https://github.com/Unidata/awips2/" title="Go to repository" class="md-source" data-md-source="github">
<div class="md-source__icon">
<svg viewBox="0 0 24 24" width="24" height="24">
<use xlink:href="#github" width="24" height="24"></use>
</svg>
</div>
<div class="md-source__repository">
GitHub
</div>
</a>
</div>
</div>
</div>
</nav>
</header>
<div class="md-container">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" data-md-level="0">
<label class="md-nav__title md-nav__title--site" for="drawer">
<div class="md-nav__button md-logo">
<i class="md-icon md-icon--home"></i>
</div>
Unidata AWIPS User Manual
</label>
<div class="md-nav__source">
<a href="https://github.com/Unidata/awips2/" title="Go to repository" class="md-source" data-md-source="github">
<div class="md-source__icon">
<svg viewBox="0 0 24 24" width="24" height="24">
<use xlink:href="#github" width="24" height="24"></use>
</svg>
</div>
<div class="md-source__repository">
GitHub
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../.." title="Home" class="md-nav__link">
Home
</a>
</li>
<li class="md-nav__item">
<a href="../../install/install-cave/" title="Install CAVE" class="md-nav__link">
Install CAVE
</a>
</li>
<li class="md-nav__item">
<a href="../../install/install-edex/" title="Install EDEX" class="md-nav__link">
Install EDEX
</a>
</li>
<li class="md-nav__item">
<a href="../../install/common-problems/" title="Common Problems" class="md-nav__link">
Common Problems
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-5" type="checkbox" id="nav-5">
<label class="md-nav__link" for="nav-5">
CAVE User Manual
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
<label class="md-nav__title" for="nav-5">
CAVE User Manual
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../cave/d2d-perspective/" title="D2D Perspective" class="md-nav__link">
D2D Perspective
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/maps-views-projections/" title="Maps, Views, Projections" class="md-nav__link">
Maps, Views, Projections
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/bundles-and-procedures/" title="Displays and Procedures" class="md-nav__link">
Displays and Procedures
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/localization-perspective/" title="Localization Perspective" class="md-nav__link">
Localization Perspective
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/nsharp/" title="NSHARP" class="md-nav__link">
NSHARP
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/warngen/" title="WarnGen Walkthrough" class="md-nav__link">
WarnGen Walkthrough
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/goes-16-17-satellite/" title="GOES 16/17" class="md-nav__link">
GOES 16/17
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-gis-shapefiles/" title="GIS and Shapefiles" class="md-nav__link">
GIS and Shapefiles
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-gridded-models/" title="Volume Browser" class="md-nav__link">
Volume Browser
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-tools/" title="Display Tools" class="md-nav__link">
Display Tools
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-radar-tools/" title="Radar Tools" class="md-nav__link">
Radar Tools
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/d2d-edit-menus/" title="Editing Menus" class="md-nav__link">
Editing Menus
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/cave-localization/" title="Change Localization" class="md-nav__link">
Change Localization
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/import-export/" title="Import/Export" class="md-nav__link">
Import/Export
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-6" type="checkbox" id="nav-6" checked>
<label class="md-nav__link" for="nav-6">
EDEX User Manual
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
<label class="md-nav__title" for="nav-6">
EDEX User Manual
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../install/start-edex/" title="EDEX Basic Commands" class="md-nav__link">
EDEX Basic Commands
</a>
</li>
<li class="md-nav__item">
<a href="../settings/" title="EDEX Settings" class="md-nav__link">
EDEX Settings
</a>
</li>
<li class="md-nav__item">
<a href="../distributed-computing/" title="Distributed EDEX" class="md-nav__link">
Distributed EDEX
</a>
</li>
<li class="md-nav__item">
<a href="../edex-ingest-docker-container/" title="Docker EDEX" class="md-nav__link">
Docker EDEX
</a>
</li>
<li class="md-nav__item">
<a href="../ldm/" title="LDM Feeds" class="md-nav__link">
LDM Feeds
</a>
</li>
<li class="md-nav__item">
<a href="../data-distribution-files/" title="Data Distribution Files" class="md-nav__link">
Data Distribution Files
</a>
</li>
<li class="md-nav__item">
<a href="../new-grid/" title="Ingest a New Grid" class="md-nav__link">
Ingest a New Grid
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="toc">
<label class="md-nav__link md-nav__link--active" for="toc">
Purging and Retention
</label>
<a href="./" title="Purging and Retention" class="md-nav__link md-nav__link--active">
Purging and Retention
</a>
<nav class="md-nav md-nav--secondary">
<label class="md-nav__title" for="toc">Table of contents</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#purge-types" title="Purge Types" class="md-nav__link">
Purge Types
</a>
</li>
<li class="md-nav__item">
<a href="#processed-data-purging" title="Processed Data Purging" class="md-nav__link">
Processed Data Purging
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#time-based-purge" title="Time-based purge" class="md-nav__link">
Time-based purge
</a>
</li>
<li class="md-nav__item">
<a href="#30-day-nexrad3-example" title="30-day NEXRAD3 Example" class="md-nav__link">
30-day NEXRAD3 Example
</a>
</li>
<li class="md-nav__item">
<a href="#frame-based-purge" title="Frame-Based Purge" class="md-nav__link">
Frame-Based Purge
</a>
</li>
<li class="md-nav__item">
<a href="#purge-logs" title="Purge Logs" class="md-nav__link">
Purge Logs
</a>
</li>
<li class="md-nav__item">
<a href="#all-purge-rules" title="All Purge Rules" class="md-nav__link">
All Purge Rules
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#raw-data-purging" title="Raw Data Purging" class="md-nav__link">
Raw Data Purging
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#default-retention" title="Default Retention" class="md-nav__link">
Default Retention
</a>
</li>
<li class="md-nav__item">
<a href="#selected-retention" title="Selected Retention" class="md-nav__link">
Selected Retention
</a>
</li>
<li class="md-nav__item">
<a href="#logging" title="Logging" class="md-nav__link">
Logging
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../edex-users/" title="Monitor Users" class="md-nav__link">
Monitor Users
</a>
</li>
<li class="md-nav__item">
<a href="../data-plugins/" title="Data Plugins" class="md-nav__link">
Data Plugins
</a>
</li>
<li class="md-nav__item">
<a href="../case-studies/" title="Archive Case Studies" class="md-nav__link">
Archive Case Studies
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../../python/overview/" title="Python-AWIPS" class="md-nav__link">
Python-AWIPS
</a>
</li>
<li class="md-nav__item">
<a href="../../appendix/educational-resources/" title="Educational Resources" class="md-nav__link">
Educational Resources
</a>
</li>
<li class="md-nav__item">
<a href="../../dev/awips-development-environment/" title="Development" class="md-nav__link">
Development
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-10" type="checkbox" id="nav-10">
<label class="md-nav__link" for="nav-10">
Appendix
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
<label class="md-nav__title" for="nav-10">
Appendix
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../appendix/appendix-grid-parameters/" title="AWIPS Grid Parameters" class="md-nav__link">
AWIPS Grid Parameters
</a>
</li>
<li class="md-nav__item">
<a href="../../python/maps-database/" title="Maps Database" class="md-nav__link">
Maps Database
</a>
</li>
<li class="md-nav__item">
<a href="../../appendix/appendix-acronyms/" title="Acronyms and Abbreviations" class="md-nav__link">
Acronyms and Abbreviations
</a>
</li>
<li class="md-nav__item">
<a href="../../appendix/appendix-wsr88d/" title="WSR-88D Product Table" class="md-nav__link">
WSR-88D Product Table
</a>
</li>
<li class="md-nav__item">
<a href="../../cave/cave-keyboard-shortcuts/" title="Keyboard Shortcuts" class="md-nav__link">
Keyboard Shortcuts
</a>
</li>
<li class="md-nav__item">
<a href="../../raytheon/cave_d2d/" title="Raytheon: CAVE User's Manual" class="md-nav__link">
Raytheon: CAVE User's Manual
</a>
</li>
<li class="md-nav__item">
<a href="../../raytheon/smm/" title="Raytheon: AWIPS System Manager's Manual" class="md-nav__link">
Raytheon: AWIPS System Manager's Manual
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary">
<label class="md-nav__title" for="toc">Table of contents</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#purge-types" title="Purge Types" class="md-nav__link">
Purge Types
</a>
</li>
<li class="md-nav__item">
<a href="#processed-data-purging" title="Processed Data Purging" class="md-nav__link">
Processed Data Purging
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#time-based-purge" title="Time-based purge" class="md-nav__link">
Time-based purge
</a>
</li>
<li class="md-nav__item">
<a href="#30-day-nexrad3-example" title="30-day NEXRAD3 Example" class="md-nav__link">
30-day NEXRAD3 Example
</a>
</li>
<li class="md-nav__item">
<a href="#frame-based-purge" title="Frame-Based Purge" class="md-nav__link">
Frame-Based Purge
</a>
</li>
<li class="md-nav__item">
<a href="#purge-logs" title="Purge Logs" class="md-nav__link">
Purge Logs
</a>
</li>
<li class="md-nav__item">
<a href="#all-purge-rules" title="All Purge Rules" class="md-nav__link">
All Purge Rules
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#raw-data-purging" title="Raw Data Purging" class="md-nav__link">
Raw Data Purging
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#default-retention" title="Default Retention" class="md-nav__link">
Default Retention
</a>
</li>
<li class="md-nav__item">
<a href="#selected-retention" title="Selected Retention" class="md-nav__link">
Selected Retention
</a>
</li>
<li class="md-nav__item">
<a href="#logging" title="Logging" class="md-nav__link">
Logging
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content">
<article class="md-content__inner md-typeset">
<a href="https://github.com/Unidata/awips2/edit/unidata_18.2.1/docs/edex/data-purge.md" title="Edit this page" class="md-icon md-content__icon">edit</a>
<h1 id="purging-and-retention">Purging and Retention<a class="headerlink" href="#purging-and-retention" title="Permanent link"></a></h1>
<h2 id="purge-types">Purge Types<a class="headerlink" href="#purge-types" title="Permanent link"></a></h2>
<p>There are two main forms of data puring in AWIPS. The most often thought of is the purging for <a href="#processed-data-purging"><strong>processed data</strong></a>. This has to do with how long data is stored for <strong>after</strong> it has been decoded and processed.</p>
<p>The second type of purging has to do with <a href="#raw-data-purging"><strong>raw data</strong></a>. This has to do with how long data is stored for <strong>before</strong> it has been decoded.</p>
<h2 id="processed-data-purging">Processed Data Purging<a class="headerlink" href="#processed-data-purging" title="Permanent link"></a></h2>
<p>AWIPS uses a plugin-based purge strategy for processed <strong>HDF5 data</strong>. This allows the user to change the purge frequency for each plugin individually, and even set purge rules for specific products for a particular plugin. There is also a default purge rules file for those products which do not have specific rules written.</p>
<blockquote>
<p><strong>Note</strong>: Purging is triggered by a quartz timer event that fires at 30 minutes after each hour. </p>
</blockquote>
<p>Purging rules are defined in XML files in the Localization Store. On EDEX, most are located in <code>/awips2/edex/data/utility/common_static/base/purge</code>, and follow the <strong>base/site</strong> localization pattern (e.g. site purge files are in <code>site/XXX/purge</code> rather than <code>base/purge</code>, where XXX is the site identifier).</p>
<p>Each data set can have a purge rule defined, and the xml file is named after the data set:</p>
<pre><code>ls /awips2/edex/data/utility/common_static/base/purge/
acarsPurgeRules.xml bufruaPurgeRules.xml pirepPurgeRules.xml
acarssoundingPurgeRules.xml ccfpPurgeRules.xml poessoundingPurgeRules.xml
aggregatePurgeRules.xml convsigmetPurgeRules.xml pointsetPurgeRules.xml
airepPurgeRules.xml cwaPurgeRules.xml profilerPurgeRules.xml
...
</code></pre>
<hr />
<h3 id="time-based-purge">Time-based purge<a class="headerlink" href="#time-based-purge" title="Permanent link"></a></h3>
<p>If a plugin has no XML file, the default rule of 1 day (24 hours) is used, from <code>/awips2/edex/data/utility/common_static/base/purge/defaultPurgeRules.xml</code>:</p>
<!--
Use html <pre> tag to be able to bold a specific section. Make sure to replace '<' with '&lt;' and '>' with '&gt;'.
-->
<pre>
&lt;purgeRuleSet&gt;
&lt;defaultRule&gt;
<b>&lt;period&gt;01-00:00:00&lt;/period&gt;</b>
&lt;/defaultRule&gt;
&lt;/purgeRuleSet&gt;
</pre>
<p>Time-based purging is set with the <em>period</em> tag and uses the <em>reference time</em> of the data. The reference time of the data is determined by the decoder. </p>
<hr />
<h3 id="30-day-nexrad3-example">30-day NEXRAD3 Example<a class="headerlink" href="#30-day-nexrad3-example" title="Permanent link"></a></h3>
<p>Modify <code>/awips2/edex/data/utility/common_static/base/purge/radarPurgeRules.xml</code> to increase the data retention period from 1 to 31 days:</p>
<pre>
&lt;purgeRuleSet&gt;
&lt;defaultRule&gt;
<b>&lt;period&gt;31-00:00:00&lt;/period&gt;</b>
&lt;/defaultRule&gt;
&lt;/purgeRuleSet&gt;
</pre>
<blockquote>
<p><strong>Note</strong>: you do NOT have to restart EDEX when you change a purge rule!</p>
</blockquote>
<hr />
<h3 id="frame-based-purge">Frame-Based Purge<a class="headerlink" href="#frame-based-purge" title="Permanent link"></a></h3>
<p>Some plugins use frame-base purging, retaining and certain number of product "versions". </p>
<p><code>/awips2/edex/data/utility/common_static/base/purge/gridPurgeRules.xml</code></p>
<pre>
<b>&lt;defaultRule&gt;
&lt;versionsToKeep&gt;2&lt;/versionsToKeep&gt;</b>
&lt;period&gt;07-00:00:00&lt;/period&gt;
&lt;/defaultRule&gt;
&lt;rule&gt;
<b>&lt;keyValue&gt;LAPS&lt;/keyValue&gt;
&lt;versionsToKeep&gt;30&lt;/versionsToKeep&gt;</b>
&lt;/rule&gt;
&lt;rule regex="true"&gt;
<b>&lt;keyValue&gt;NAM(?:12|20|40)&lt;/keyValue&gt;
&lt;versionsToKeep&gt;2&lt;/versionsToKeep&gt;
&lt;modTimeToWait&gt;00-00:15:00&lt;/modTimeToWait&gt;</b>
&lt;/rule&gt;
...
</pre>
<p>In the above example, notice a <em>default rule</em> (2) is specified, as well as specific models with their own rules.<br />
The tag <em>modTimeToWait</em> can be used in conjunction with <em>versionsToKeep</em> and will increase the versionsToKeep by 1 if data matching this rule has been stored within modTimeToWait.</p>
<hr />
<h3 id="purge-logs">Purge Logs<a class="headerlink" href="#purge-logs" title="Permanent link"></a></h3>
<p>Data purge events are logged to the file <code>edex-ingest-purge-[yyyymmdd].log</code>, where <code>[yyyymmdd]</code> is the date stamp. </p>
<pre><code>tail -f edex-ingest-purge-20120327.log
--------START LOG PURGE---------
INFO 2012-03-27 00:30:00,027 [DefaultQuartzScheduler_Worker-3] PurgeLogger: EDEX - PURGE LOGS::Skipped file with invalid fileName: afos-trigger.log
INFO 2012-03-27 00:30:00,193 [DefaultQuartzScheduler_Worker-3] PurgeLogger: EDEX - PURGE LOGS::Removed 1 old files
INFO 2012-03-27 00:31:23,155 [DefaultQuartzScheduler_Worker-3] PurgeLogger: EDEX - PURGE LOGS::Archived 14 files
INFO 2012-03-27 00:31:23,155 [DefaultQuartzScheduler_Worker-3] PurgeLogger: EDEX - PURGE LOGS::Skipped processing 1 files
INFO 2012-03-27 00:31:23,155 [DefaultQuartzScheduler_Worker-3] PurgeLogger: EDEX - PURGE LOGS::---------END LOG PURGE-----------
</code></pre>
<hr />
<h3 id="all-purge-rules">All Purge Rules<a class="headerlink" href="#all-purge-rules" title="Permanent link"></a></h3>
<p>To see all purge rule directories (base, site, configured):</p>
<pre><code>find /awips2/edex/data/utility -name purge
/awips2/edex/data/utility/common_static/base/purge
</code></pre>
<p>If any overrides have been made, then it's possible that <em>site</em> directories may show up as results from the find command as well.</p>
<hr />
<h2 id="raw-data-purging">Raw Data Purging<a class="headerlink" href="#raw-data-purging" title="Permanent link"></a></h2>
<p>Raw data are files that have been brought in by the LDM and recognized by an action in the <strong>pqact.conf</strong> file. These files are written to subdirectories of <code>/awips2/data_store/</code>. This data will wait here until it is purged, from the purging rules defined in <code>/awips2/edex/data/utility/common_static/base/archiver/purger/RAW_DATA.xml</code>.</p>
<p>If the purge time is too short, and the processing latencies on EDEX are too long, it is possible that EDEX will miss some of this data, and the purge times will need to be adjusted by changing the <a href="#default-retention"><code>&lt;defaultRetentionHours&gt;</code></a> or <a href="#selected-retention"><code>&lt;selectedRetentionHours&gt;</code></a> tag on the relevent data sets.</p>
<hr />
<h3 id="default-retention">Default Retention<a class="headerlink" href="#default-retention" title="Permanent link"></a></h3>
<p>The <strong>defaultRetentionHours</strong> tag is defined at the beginning of the RAW_DATA.xml file. It is the duration that will apply to any piece of data that does not fall under an explicitly defined <a href="#selected-retention">category</a>.</p>
<p>The default value for our EDEX is 1 hour:</p>
<pre>
&lt;archive&gt;
&lt;name&gt;Raw&lt;/name&gt;
&lt;rootDir&gt;/awips2/data_store/&lt;/rootDir&gt;
<b>&lt;defaultRetentionHours&gt;1&lt;/defaultRetentionHours&gt;</b>
&lt;category&gt;
...
</pre>
<hr />
<h3 id="selected-retention">Selected Retention<a class="headerlink" href="#selected-retention" title="Permanent link"></a></h3>
<p>Data sets are broken up into <em>categories</em> in the RAW_DATA.xml file. These categories are groupings of similar data. Each category has a <strong>selectedRetentionHours</strong> tag which specifies how long the matching data will be kept for.</p>
<p>For example, there is a <strong>Model</strong> category which sets the purge time to 3 hours for all grib, bufrmos, and modelsounding data:</p>
<pre>
...
<b>&lt;category&gt;
&lt;name&gt;Model&lt;/name&gt;
&lt;selectedRetentionHours&gt;3&lt;/selectedRetentionHours&gt;</b>
&lt;dataSet&gt;
<b>&lt;dirPattern&gt;(grib|grib2)/(\d{4})(\d{2})(\d{2})/(\d{2})/(.*)&lt;/dirPattern&gt;</b>
&lt;displayLabel&gt;{1} - {6}&lt;/displayLabel&gt;
&lt;dateGroupIndices&gt;2,3,4,5&lt;/dateGroupIndices&gt;
&lt;/dataSet&gt;
&lt;dataSet&gt;
<b>&lt;dirPattern&gt;(bufrmos|modelsounding)/(\d{4})(\d{2})(\d{2})/(\d{2})&lt;/dirPattern&gt;</b>
&lt;displayLabel&gt;{1}&lt;/displayLabel&gt;
&lt;dateGroupIndices&gt;2,3,4,5&lt;/dateGroupIndices&gt;
&lt;/dataSet&gt;
&lt;/category&gt;
...
</pre>
<h3 id="logging">Logging<a class="headerlink" href="#logging" title="Permanent link"></a></h3>
<p>Raw data purging can be seen in the <strong>purge</strong> logs as well (<code>/awips2/edex/logs/edex-ingest-purge-[yyyymmdd].log</code> where <code>[yyyymmdd]</code> is the date stamp).</p>
<pre><code>[centos@tg-atm160027-edex-dev purge]$ grep -i 'archive' /awips2/edex/logs/edex-ingest-purge-20200728.log
INFO 2020-07-28 20:05:23,959 2329 [Purge-Archive] ArchivePurgeManager: EDEX - Start purge of category Raw - Observation, directory "/awips2/data_store/bufrhdw".
INFO 2020-07-28 20:05:23,960 2330 [Purge-Archive] ArchivePurgeManager: EDEX - End purge of category Raw - Observation, directory "/awips2/data_store/bufrhdw", deleted 0 files and directories.
INFO 2020-07-28 20:05:23,961 2331 [Purge-Archive] ArchivePurgeManager: EDEX - Unlocked: "/awips2/data_store/bufrhdw"
INFO 2020-07-28 20:05:23,963 2332 [Purge-Archive] ArchivePurgeManager: EDEX - Locked: "/awips2/data_store/xml"
INFO 2020-07-28 20:05:23,963 2333 [Purge-Archive] ArchivePurgeManager: EDEX - Start purge of category Raw - Products, directory "/awips2/data_store/xml".
INFO 2020-07-28 20:05:23,964 2334 [Purge-Archive] ArchivePurgeManager: EDEX - End purge of category Raw - Products, directory "/awips2/data_store/xml", deleted 5 files and directories.
INFO 2020-07-28 20:05:23,967 2335 [Purge-Archive] ArchivePurgeManager: EDEX - Unlocked: "/awips2/data_store/xml"
INFO 2020-07-28 20:05:23,967 2336 [Purge-Archive] ArchivePurger: EDEX - Raw::Archive Purged 28387 files in 23.8s.
INFO 2020-07-28 20:05:23,979 2337 [Purge-Archive] ArchivePurgeManager: EDEX - Purging directory: "/awips2/edex/data/archive".
INFO 2020-07-28 20:05:23,992 2338 [Purge-Archive] ArchivePurger: EDEX - Processed::Archive Purged 0 files in 25ms.
INFO 2020-07-28 20:05:23,992 2339 [Purge-Archive] ArchivePurger: EDEX - Archive Purge finished. Time to run: 23.9s
...
</code></pre>
</article>
</div>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-nav">
<nav class="md-footer-nav__inner md-grid">
<a href="../new-grid/" title="Ingest a New Grid" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
<div class="md-flex__cell md-flex__cell--shrink">
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
</div>
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
<span class="md-flex__ellipsis">
<span class="md-footer-nav__direction">
Previous
</span>
Ingest a New Grid
</span>
</div>
</a>
<a href="../edex-users/" title="Monitor Users" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
<span class="md-flex__ellipsis">
<span class="md-footer-nav__direction">
Next
</span>
Monitor Users
</span>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
</div>
</a>
</nav>
</div>
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-footer-copyright">
<div class="md-footer-copyright__highlight">
Copyright &copy; 2020 Unidata Program Center.
</div>
powered by
<a href="http://www.mkdocs.org" title="MkDocs">MkDocs</a>
and
<a href="http://squidfunk.github.io/mkdocs-material/" title="Material for MkDocs">
Material for MkDocs</a>
</div>
<div class="md-footer-social">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
<a href="https://www.unidata.ucar.edu/software/awips2" class="md-footer-social__link fa fa-globe"></a>
<a href="https://github.com/Unidata/awips2" class="md-footer-social__link fa fa-github-alt"></a>
<a href="https://twitter.com/Unidata" class="md-footer-social__link fa fa-twitter"></a>
<a href="https://facebook.com/Unidata" class="md-footer-social__link fa fa-facebook"></a>
</div>
</div>
</div>
</footer>
</div>
<script src="../../assets/javascripts/application-0b7df094bf.js"></script>
<script>app.initialize({url:{base:"../.."}})</script>
<script src="../../assets/javascripts/application-0b7df094bf.js"></script>
</body>
</html>
Reference in a new issue View git blame Copy permalink