From f12fb4213045494de98b7f18e3cf762cc943fd94 Mon Sep 17 00:00:00 2001 From: Ben Steffensmeier Date: Wed, 31 Oct 2012 15:35:51 -0500 Subject: [PATCH] Issue #189 fix the grid deltaScripts to work with static data and fix rollback Former-commit-id: b71fed55785cb0f49cb862cfe6dcb9c1ed4f7541 [formerly 05e3b37896f8e0fee679269d00a7b5eb0e787b1e] [formerly b5f84376f0c91cda869659843089be7becbee015] [formerly b71fed55785cb0f49cb862cfe6dcb9c1ed4f7541 [formerly 05e3b37896f8e0fee679269d00a7b5eb0e787b1e] [formerly b5f84376f0c91cda869659843089be7becbee015] [formerly c68278813094224a0576ece21cc973e13c02e28f [formerly b5f84376f0c91cda869659843089be7becbee015 [formerly 79dca17198b19df77c7eb1074e492a16516efc22]]]] Former-commit-id: c68278813094224a0576ece21cc973e13c02e28f Former-commit-id: 8ca1571504d3d599bca24d9d2d84da72d18a761f [formerly 29d0d612bde5bffbe7b403d191f39e30cd9d54e7] [formerly c12a67897e4ed7b061af78ce017a79541e5cad82 [formerly b5431d9ccfc57abdf1784d7de75ddafd094a6265]] Former-commit-id: 7747c00e85c7fc4b0e0e77049ed2f1b65654ed32 [formerly d1a5ee130fe84aa8c17c27699c9cb3651f73941e] Former-commit-id: db55f939fb2f7969c26eb49730fbdfa260626792 --- .../unified_grid/convert_grib_data.py | 34 ++++++++-------- .../convert_grib_data.py | 40 +++++++++---------- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/deltaScripts/unified_grid/convert_grib_data.py b/deltaScripts/unified_grid/convert_grib_data.py index e08904d996..8f0df9e9a7 100644 --- a/deltaScripts/unified_grid/convert_grib_data.py +++ b/deltaScripts/unified_grid/convert_grib_data.py @@ -97,8 +97,6 @@ def convertModel(modelName): secondaryId = "Version" + version ensembleId = convertPert(pert) newdatauri = "/grid/" + datatime + "/" + modelName + "/" + secondaryId + "/" + ensembleId + "/" + gridcoverageid + "/" + paramabbrev + "/" + masterlevel + "/" + levelone + "/" + leveltwo - if paramabbrev.startswith("static") and gribforecasttime != 0: - continue hdfTime -= time() try: forecast = int(gribforecasttime)/3600 @@ -106,26 +104,26 @@ def convertModel(modelName): newgrp = newdatauri dataset="Data" if paramabbrev.startswith("static"): - forecast = 0 prevgrp = "/" - newgrp = "/" + gridcoveragename + newgrp = "/" + gridcoverageid dataset=paramabbrev filebase = "/%s-%s-FH-%.3d.h5" % (modelName, gribreftime.split(":")[0].replace(" ", "-"), forecast) hdf5file = gridFiles + masterlevel + filebase - if lastFile != None and lastFile.filename != hdf5file: - #print "Closing", lastFile.filename - lastFile.close() - lastFile = None - if lastFile == None: - if not(exists(hdf5file)): - t0 = time() - if not(isdir(gridFiles + masterlevel)): - mkdir(gridFiles + masterlevel) - move(gribFiles + masterlevel + filebase, gridFiles + masterlevel) - hdfTime -= (time() - t0) - #print "Opening", hdf5file - lastFile = h5py.File(hdf5file) - copyH5(lastFile, prevgrp, newgrp, dataset) + if not(paramabbrev.startswith("static")) or forecast == 0: + if lastFile != None and lastFile.filename != hdf5file: + #print "Closing", lastFile.filename + lastFile.close() + lastFile = None + if lastFile == None: + if not(exists(hdf5file)): + t0 = time() + if not(isdir(gridFiles + masterlevel)): + mkdir(gridFiles + masterlevel) + move(gribFiles + masterlevel + filebase, gridFiles + masterlevel) + hdfTime -= (time() - t0) + #print "Opening", hdf5file + lastFile = h5py.File(hdf5file) + copyH5(lastFile, prevgrp, newgrp, dataset) except: print modelName, "Error", gribdatauri print sys.exc_info()[1] diff --git a/deltaScripts/unified_grid_rollback/convert_grib_data.py b/deltaScripts/unified_grid_rollback/convert_grib_data.py index 7e75116b1b..afb14740d9 100644 --- a/deltaScripts/unified_grid_rollback/convert_grib_data.py +++ b/deltaScripts/unified_grid_rollback/convert_grib_data.py @@ -87,8 +87,6 @@ def convertModel(modelName): secondaryId = "Version" + version ensembleId = convertPert(pert) newdatauri = "/grid/" + datatime + "/" + modelName + "/" + secondaryId + "/" + ensembleId + "/" + gridcoverageid + "/" + paramabbrev + "/" + masterlevel + "/" + levelone + "/" + leveltwo - if paramabbrev.startswith("static") and gribforecasttime != 0: - continue hdfTime -= time() try: forecast = int(gribforecasttime)/3600 @@ -96,26 +94,26 @@ def convertModel(modelName): newgrp = newdatauri dataset="Data" if paramabbrev.startswith("static"): - forecast = 0 - prevgrp = "/" - newgrp = "/" + gridcoveragename + prevgrp = "/"+ gridcoverageid + newgrp = "/" dataset=paramabbrev - filebase = "/%s-%s-FH-%.3d.h5" % (modelName, gribreftime.split(":")[0].replace(" ", "-"), forecast) - hdf5file = gribFiles + masterlevel + filebase - if lastFile != None and lastFile.filename != hdf5file: - #print "Closing", lastFile.filename - lastFile.close() - lastFile = None - if lastFile == None: - if not(exists(hdf5file)): - t0 = time() - if not(isdir(gribFiles + masterlevel)): - mkdir(gribFiles + masterlevel) - move(gridFiles + masterlevel + filebase, gribFiles) - hdfTime -= (time() - t0) - #print "Opening", hdf5file - lastFile = h5py.File(hdf5file) - copyH5(lastFile, newgrp, prevgrp, dataset) + if not(paramabbrev.startswith("static")) or forecast == 0: + filebase = "/%s-%s-FH-%.3d.h5" % (modelName, gribreftime.split(":")[0].replace(" ", "-"), forecast) + hdf5file = gribFiles + masterlevel + filebase + if lastFile != None and lastFile.filename != hdf5file: + #print "Closing", lastFile.filename + lastFile.close() + lastFile = None + if lastFile == None: + if not(exists(hdf5file)): + t0 = time() + if not(isdir(gribFiles + masterlevel)): + mkdir(gribFiles + masterlevel) + move(gridFiles + masterlevel + filebase, gribFiles + masterlevel) + hdfTime -= (time() - t0) + #print "Opening", hdf5file + lastFile = h5py.File(hdf5file) + copyH5(lastFile, newgrp, prevgrp, dataset) except: print modelName, "Error", gribdatauri print sys.exc_info()[1]