From 1f731e8b256f39f0abc3003acb536610a7435a5a Mon Sep 17 00:00:00 2001 From: Bill Ladwig Date: Wed, 24 Aug 2016 17:03:51 -0600 Subject: [PATCH] Added the jinja patch for 1.21 conda-build so it works with numpy distutils. Added code for the threadlocal caching. Added a cached test. Modified cloudfrac to match what has been committed to NCL. Modified the NCL test script to add cloud fraction. --- conda_recipe/meta.yaml | 9 +- .../{cloud_fracf.f90 => wrf_cloud_fracf.f90} | 0 fortran/wrffortran.pyf | 22 +-- ncl_reference/WRFUserARW.ncl | 152 ++++++++++----- ncl_reference/jinja_context.py | 183 ++++++++++++++++++ setup.py | 4 +- src/wrf/cache.py | 52 +++++ src/wrf/config.py | 76 +++++--- src/wrf/metadecorators.py | 14 +- src/wrf/routines.py | 3 +- src/wrf/util.py | 49 ++--- test/cachetest.py | 65 +++++++ test/ncl_get_var.ncl | 2 +- test/utests.py | 47 ++--- 14 files changed, 521 insertions(+), 157 deletions(-) rename fortran/{cloud_fracf.f90 => wrf_cloud_fracf.f90} (100%) create mode 100644 ncl_reference/jinja_context.py create mode 100644 src/wrf/cache.py create mode 100644 test/cachetest.py diff --git a/conda_recipe/meta.yaml b/conda_recipe/meta.yaml index 3e1def3..f8d1a9e 100644 --- a/conda_recipe/meta.yaml +++ b/conda_recipe/meta.yaml @@ -1,13 +1,16 @@ + +# For the version to work, the patch in ncl_reference needs to be applied +# to the installed conda-build. Otherwise, it needs to be set manually until +# continuum can release a fix. package: name: wrf-python - version: "0.0.1" + version: {{ load_setup_py_data().version }} source: git_url: git@github.com:NCAR/wrf-python.git - git_rev: prebeta.0.1 build: - number: 5 + number: 1 detect_binary_files_with_prefix: true requirements: diff --git a/fortran/cloud_fracf.f90 b/fortran/wrf_cloud_fracf.f90 similarity index 100% rename from fortran/cloud_fracf.f90 rename to fortran/wrf_cloud_fracf.f90 diff --git a/fortran/wrffortran.pyf b/fortran/wrffortran.pyf index 831eea3..1c652a5 100644 --- a/fortran/wrffortran.pyf +++ b/fortran/wrffortran.pyf @@ -22,17 +22,6 @@ python module _wrffortran ! in real(kind=8) dimension(nx,ny,nz),intent(inout),depend(nx,ny,nz) :: tem1 real(kind=8) dimension(nx,ny,nz),intent(inout),depend(nx,ny,nz) :: tem2 end subroutine dcalcuh - subroutine dcloudfrac(pres,rh,lowc,midc,highc,nz,ns,ew) ! in :_wrffortran:cloud_fracf.f90 - threadsafe - real(kind=8) dimension(ew,ns,nz),intent(in) :: pres - real(kind=8) dimension(ew,ns,nz),intent(in),depend(ew,ns,nz) :: rh - real(kind=8) dimension(ew,ns),intent(out,in),depend(ew,ns) :: lowc - real(kind=8) dimension(ew,ns),intent(out,in),depend(ew,ns) :: midc - real(kind=8) dimension(ew,ns),intent(out,in),depend(ew,ns) :: highc - integer, optional,check(shape(pres,2)==nz),depend(pres) :: nz=shape(pres,2) - integer, optional,check(shape(pres,1)==ns),depend(pres) :: ns=shape(pres,1) - integer, optional,check(shape(pres,0)==ew),depend(pres) :: ew=shape(pres,0) - end subroutine dcloudfrac subroutine deqthecalc(qvp,tmk,prs,eth,miy,mjx,mkzh) ! in :_wrffortran:eqthecalc.f90 threadsafe use wrf_constants, only: tlclc2,tlclc3,tlclc1,tlclc4,eps,thtecon3,gammamd,thtecon1,gamma,thtecon2 @@ -104,6 +93,17 @@ python module _wrffortran ! in integer intent(inout) :: errstat character*(*) intent(inout) :: errmsg end subroutine dcapecalc3d + subroutine dcloudfrac(pres,rh,lowc,midc,highc,nz,ns,ew) ! in :_wrffortran:wrf_cloud_fracf.f90 + threadsafe + real(kind=8) dimension(ew,ns,nz),intent(in) :: pres + real(kind=8) dimension(ew,ns,nz),intent(in),depend(ew,ns,nz) :: rh + real(kind=8) dimension(ew,ns),intent(out,in),depend(ew,ns) :: lowc + real(kind=8) dimension(ew,ns),intent(out,in),depend(ew,ns) :: midc + real(kind=8) dimension(ew,ns),intent(out,in),depend(ew,ns) :: highc + integer, optional,check(shape(pres,2)==nz),depend(pres) :: nz=shape(pres,2) + integer, optional,check(shape(pres,1)==ns),depend(pres) :: ns=shape(pres,1) + integer, optional,check(shape(pres,0)==ew),depend(pres) :: ew=shape(pres,0) + end subroutine dcloudfrac module wrf_constants ! in :_wrffortran:wrf_constants.f90 real(kind=8), parameter,optional :: wrf_earth_radius=6370000.d0 real(kind=8), parameter,optional :: rhowat=1000.d0 diff --git a/ncl_reference/WRFUserARW.ncl b/ncl_reference/WRFUserARW.ncl index e2cc410..647e262 100755 --- a/ncl_reference/WRFUserARW.ncl +++ b/ncl_reference/WRFUserARW.ncl @@ -894,7 +894,7 @@ begin uvmet(1,:,:) = (/v(:,:)/) end if copy_VarAtts_except(u,uvmet,(/"description","units"/)) - uvmet@description = " u,v met velocity" + uvmet@description = "u,v met velocity" uvmet!0 = "u_v" do n=0,dimsizes(dims)-1 ; Copy dimension names if(isdimnamed(u,n)) @@ -950,7 +950,7 @@ begin end if if( (variable .eq. "uvmet10") ) then - uvmet@description = " u10,v10 met velocity" + uvmet@description = "u10,v10 met velocity" end if return(uvmet) @@ -1377,8 +1377,69 @@ begin return(cape) end if + if( any( variable .eq. "cfrac" )) then + + print("calculating wrf cloud fraction") + if(typeof(file_handle).eq."file") then + ISFILE = True + nc_file = file_handle + else if(typeof(file_handle).eq."list") then + ISFILE = False + nc_file = file_handle[0] + else + print("wrf_user_getvar: error: the first argument must be a file or a list of files opened with addfile or addfiles") + return + end if + end if + + time_in = time + if ( time .eq. -1 ) then + if(ISFILE) then + P = nc_file->P + PB = nc_file->PB + rh = wrf_user_getvar(nc_file,"rh",-1) + else + P = file_handle[:]->P + PB = file_handle[:]->PB + rh = wrf_user_getvar(file_handle,"rh",-1) + end if + else + if(ISFILE) then + P = nc_file->P(time_in,:,:,:) + PB = nc_file->PB(time_in,:,:,:) + rh = wrf_user_getvar(file_handle,"rh",time_in) + else + P = file_handle[:]->P(time_in,:,:,:) + PB = file_handle[:]->PB(time_in,:,:,:) + rh = wrf_user_getvar(file_handle,"rh",time_in) + end if + end if + pres = (P + PB) + thedims = dimsizes(pres) + + + if(time .eq. -1) then + nt = thedims(0) + nz = thedims(1) + ns = thedims(2) + ew = thedims(3) + else + nt = 1 + nz = thedims(0) + ns = thedims(1) + ew = thedims(2) + end if + + out_array = wrf_cloud_fraction(pres,rh) + + out_array@units = "%" + out_array@description = "Low, Mid, High Clouds" + return(out_array) + + end if ;variable is cfrac + if( any( variable .eq. (/"pw"/) ) ) then ;Precipitable Water print("calculating precipitable water") @@ -2172,6 +2233,7 @@ begin field_type = str_lower(get_res_value_keep(opts,"field_type","none")) log_of_Pressure = get_res_value_keep(opts,"logP",False) debug = get_res_value_keep(opts,"debug",False) + timeidx = get_res_value_keep(opts,"time",-1) valid_field_types = (/"none","pressure","pres","p","z","t","ght"/) if(.not.any(field_type.eq.valid_field_types)) then @@ -2237,29 +2299,23 @@ begin ;We will need some basic fields for the interpolation ;regardless of the field requested. Get all time periods ;of the fields. - P = _get_wrf_var(file_handle,"P",-1) + _get_wrf_var(file_handle,"PB",-1) - qvp = _get_wrf_var(file_handle,"QVAPOR",-1) - terht = _get_wrf_var(file_handle,"HGT",-1) - sfp = _get_wrf_var(file_handle,"PSFC",-1) * 0.01 + P = _get_wrf_var(file_handle,"P",timeidx) + _get_wrf_var(file_handle,"PB",timeidx) + qvp = _get_wrf_var(file_handle,"QVAPOR",timeidx) + terht = _get_wrf_var(file_handle,"HGT",timeidx) + sfp = _get_wrf_var(file_handle,"PSFC",timeidx) * 0.01 Pdims = dimsizes(P) - if(ISFILE) then - ght = wrf_user_getvar(nc_file,"height",-1) - tk = wrf_user_getvar(nc_file,"tk",-1) - else - tmpz = file_handle[:]->PH - PHB = file_handle[:]->PHB - tmpz = (tmpz + PHB)/9.81 - ght = wrf_user_unstagger(tmpz,"Z") - T = file_handle[:]->T - T = T + 300. - tk = wrf_tk( P , T ) - end if - smsfp = sfp - wrf_smooth_2d(smsfp,3) + ght = wrf_user_getvar(file_handle,"height",timeidx) + tk = wrf_user_getvar(file_handle,"tk",timeidx) + smsfp = sfp + wrf_smooth_2d(smsfp,3) ;Initialize an array for the vertical coordinate - ntimes = Pdims(0) + if (timeidx .lt. 0) then + ntimes = Pdims(0) + else + ntimes = 1 + end if ;Get the vertical coordinate type vcor = 0 @@ -2276,14 +2332,26 @@ begin if(vert_coordinate .eq. "ght_agl") then vcor = 3 - rtemp = new( (/nz,ns,ew/),float) - vcord_array = new((/ntimes,nz,ns,ew/),float) - do it = 0, ntimes - 1 - do ilev = 0,nz-1 - rtemp(ilev,:,:) = ght(it,ilev,:,:) - terht(0,:,:) - end do - vcord_array(it,:,:,:) = exp(-rtemp/sclht) - end do + + if (timeidx .lt. 0) then + rtemp = new( (/nz,ns,ew/),float) + vcord_array = new((/ntimes,nz,ns,ew/),float) + do it = 0, ntimes - 1 + do ilev = 0,nz-1 + rtemp(ilev,:,:) = ght(it,ilev,:,:) - terht(it,:,:) + end do + vcord_array(it,:,:,:) = exp(-rtemp/sclht) + end do + else + rtemp = new((/nz,ns,ew/),float) + vcord_array = new((/nz,ns,ew/), float) + + do ilev = 0,nz-1 + rtemp(ilev,:,:) = ght(ilev,:,:) - terht(:,:) + end do + vcord_array(:,:,:) = exp(-rtemp/sclht) + end if + delete(rtemp) end if @@ -2291,14 +2359,9 @@ begin vcor = 4 idir = 1 icorsw = 0 - delta = 0.01 - if(ISFILE) then - coriolis = nc_file->F(0,:,:) - theta = wrf_user_getvar(nc_file,"theta",-1) - else - coriolis = file_handle[0]->F(0,:,:) - theta = T - end if + delta = 0.01 + coriolis = _get_wrf_var(file_handle, "F", timeidx) + theta = wrf_user_getvar(file_handle,"theta",timeidx) preshPa = P * 0.01 vcord_array = wrf_monotonic(theta,preshPa,coriolis,idir,delta,icorsw) ; @@ -2313,13 +2376,8 @@ begin icorsw = 0 idir = 1 delta = 0.01 - if(ISFILE) then - coriolis = nc_file->F(0,:,:) - eqpot = wrf_user_getvar(nc_file,"eth",-1) - else - coriolis = file_handle[0]->F(0,:,:) - eqpot = wrf_eth ( qvp, tk, P ) - end if + coriolis = _get_wrf_var(file_handle, "F", timeidx) + eqpot = wrf_user_getvar(file_handle,"eth",timeidx) preshPa = P * 0.01 vcord_array = wrf_monotonic(eqpot,preshPa,coriolis,idir,delta,icorsw) @@ -2331,8 +2389,8 @@ begin if(debug) then print("icase = " + icase + " extrap = " + extrap + " vcor = " + vcor + " logP = " + logP) end if - - field_out = wrf_vintrp(field,P,tk,qvp,ght,terht(0,:,:),sfp,smsfp,\ + + field_out = wrf_vintrp(field,P,tk,qvp,ght,terht,sfp,smsfp,\ vcord_array,interp_levels,icase,extrap,vcor,logP) ; Add metadata to return array @@ -2341,7 +2399,7 @@ begin ; Add new levels as a coordinate array lev_field = num_field_dims-3 field_out!lev_field = "interp_levels" - field_out&$field_out!lev_field$ = interp_levels(::-1) + field_out&$field_out!lev_field$ = interp_levels(:) field_out@vert_interp_type = vert_coordinate return(field_out) diff --git a/ncl_reference/jinja_context.py b/ncl_reference/jinja_context.py new file mode 100644 index 0000000..b1c48bc --- /dev/null +++ b/ncl_reference/jinja_context.py @@ -0,0 +1,183 @@ +''' +Created on Jan 16, 2014 + +@author: sean +''' +from __future__ import absolute_import, division, print_function + +from functools import partial +import json +import logging +import os +import sys + +import jinja2 + +from .conda_interface import PY3 +from .environ import get_dict as get_environ +from .metadata import select_lines, ns_cfg +from .source import WORK_DIR + +log = logging.getLogger(__file__) + + +class UndefinedNeverFail(jinja2.Undefined): + """ + A class for Undefined jinja variables. + This is even less strict than the default jinja2.Undefined class, + because it permits things like {{ MY_UNDEFINED_VAR[:2] }} and + {{ MY_UNDEFINED_VAR|int }}. This can mask lots of errors in jinja templates, so it + should only be used for a first-pass parse, when you plan on running a 'strict' + second pass later. + """ + all_undefined_names = [] + + def __init__(self, hint=None, obj=jinja2.runtime.missing, name=None, + exc=jinja2.exceptions.UndefinedError): + UndefinedNeverFail.all_undefined_names.append(name) + jinja2.Undefined.__init__(self, hint, obj, name, exc) + + __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \ + __truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \ + __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ + __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = \ + __complex__ = __pow__ = __rpow__ = \ + lambda self, *args, **kwargs: UndefinedNeverFail(hint=self._undefined_hint, + obj=self._undefined_obj, + name=self._undefined_name, + exc=self._undefined_exception) + + __str__ = __repr__ = \ + lambda *args, **kwargs: u'' + + __int__ = lambda _: 0 + __float__ = lambda _: 0.0 + + def __getattr__(self, k): + try: + return object.__getattr__(self, k) + except AttributeError: + return UndefinedNeverFail(hint=self._undefined_hint, + obj=self._undefined_obj, + name=self._undefined_name + '.' + k, + exc=self._undefined_exception) + + +class FilteredLoader(jinja2.BaseLoader): + """ + A pass-through for the given loader, except that the loaded source is + filtered according to any metadata selectors in the source text. + """ + + def __init__(self, unfiltered_loader): + self._unfiltered_loader = unfiltered_loader + self.list_templates = unfiltered_loader.list_templates + + def get_source(self, environment, template): + contents, filename, uptodate = self._unfiltered_loader.get_source(environment, + template) + return select_lines(contents, ns_cfg()), filename, uptodate + + +def load_setup_py_data(setup_file='setup.py', from_recipe_dir=False, recipe_dir=None, + unload_modules=None, fail_on_error=False): + + _setuptools_data = {} + + def setup(**kw): + _setuptools_data.update(kw) + + import setuptools + import distutils.core + + try: + import numpy.distutils.core + except ImportError: + do_numpy = False + else: + do_numpy = True + + cd_to_work = False + + if from_recipe_dir and recipe_dir: + setup_file = os.path.abspath(os.path.join(recipe_dir, setup_file)) + elif os.path.exists(WORK_DIR): + cd_to_work = True + cwd = os.getcwd() + os.chdir(WORK_DIR) + if not os.path.isabs(setup_file): + setup_file = os.path.join(WORK_DIR, setup_file) + # this is very important - or else if versioneer or otherwise is in the start folder, + # things will pick up the wrong versioneer/whatever! + sys.path.insert(0, WORK_DIR) + else: + log.debug("Did not find setup.py file in manually specified location, and source " + "not downloaded yet.") + return {} + + # Patch setuptools, distutils + setuptools_setup = setuptools.setup + distutils_setup = distutils.core.setup + setuptools.setup = distutils.core.setup = setup + + if do_numpy: + numpy_setup = numpy.distutils.core.setup + numpy.distutils.core.setup = setup + + ns = { + '__name__': '__main__', + '__doc__': None, + '__file__': setup_file, + } + try: + code = compile(open(setup_file).read(), setup_file, 'exec', dont_inherit=1) + exec(code, ns, ns) + distutils.core.setup = distutils_setup + setuptools.setup = setuptools_setup + if do_numpy: + numpy.distutils.core.setup = numpy_setup + # this happens if setup.py is used in load_setup_py_data, but source is not yet downloaded + except: + raise + finally: + if cd_to_work: + os.chdir(cwd) + del sys.path[0] + + return _setuptools_data + + +def load_setuptools(setup_file='setup.py', from_recipe_dir=False, recipe_dir=None, + unload_modules=None, fail_on_error=False): + log.warn("Deprecation notice: the load_setuptools function has been renamed to " + "load_setup_py_data. load_setuptools will be removed in a future release.") + return load_setup_py_data(setup_file=setup_file, from_recipe_dir=from_recipe_dir, + recipe_dir=recipe_dir, unload_modules=unload_modules, + fail_on_error=fail_on_error) + + +def load_npm(): + # json module expects bytes in Python 2 and str in Python 3. + mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'} + with open('package.json', **mode_dict) as pkg: + return json.load(pkg) + + +def context_processor(initial_metadata, recipe_dir): + """ + Return a dictionary to use as context for jinja templates. + + initial_metadata: Augment the context with values from this MetaData object. + Used to bootstrap metadata contents via multiple parsing passes. + """ + ctx = get_environ(m=initial_metadata) + environ = dict(os.environ) + environ.update(get_environ(m=initial_metadata)) + + ctx.update( + load_setup_py_data=partial(load_setup_py_data, recipe_dir=recipe_dir), + # maintain old alias for backwards compatibility: + load_setuptools=partial(load_setuptools, recipe_dir=recipe_dir), + load_npm=load_npm, + environ=environ) + return ctx diff --git a/setup.py b/setup.py index 09b7144..afa319d 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ ext1 = numpy.distutils.core.Extension( "fortran/wrf_testfunc.f90", "fortran/wrf_user.f90", "fortran/rip_cape.f90", - "fortran/cloud_fracf.f90", + "fortran/wrf_cloud_fracf.f90", "fortran/wrf_fctt.f90", "fortran/wrf_user_dbz.f90", "fortran/wrf_relhl.f90", @@ -22,7 +22,7 @@ ext1 = numpy.distutils.core.Extension( "fortran/wrffortran.pyf"] ) -with open('src/wrf/version.py') as f: +with open("src/wrf/version.py") as f: exec(f.read()) requirements = [ diff --git a/src/wrf/cache.py b/src/wrf/cache.py new file mode 100644 index 0000000..3eb5d7a --- /dev/null +++ b/src/wrf/cache.py @@ -0,0 +1,52 @@ +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +from threading import local +from collections import OrderedDict + +from .config import get_cache_size + +_local_storage = local() + +def cache_item(key, product, value): + global _local_storage + + try: + cache = _local_storage.cache + except AttributeError: + _local_storage.cache = OrderedDict() + cache = _local_storage.cache + + try: + prod_dict = cache[key] + except KeyError: + if len(cache) >= get_cache_size(): + cache.popitem(last=False) # Remove the oldest dataset + + cache[key] = OrderedDict() + prod_dict = cache[key] + + cache[key][product] = value + + +def get_cached_item(key, product): + cache = getattr(_local_storage, "cache", None) + + if cache is None: + return None + + prod_dict = cache.get(key, None) + + if prod_dict is None: + return None + + return prod_dict.get(product, None) + + +def _get_cache(): + return getattr(_local_storage, "cache", None) + + + + + \ No newline at end of file diff --git a/src/wrf/config.py b/src/wrf/config.py index 642685e..973dc50 100644 --- a/src/wrf/config.py +++ b/src/wrf/config.py @@ -4,75 +4,87 @@ from __future__ import (absolute_import, division, print_function, try: from xarray import DataArray except ImportError: - _XARRAY_ENABLED = False + _xarray_enabled = False else: - _XARRAY_ENABLED = True + _xarray_enabled = True try: from cartopy import crs except ImportError: - _CARTOPY_ENABLED = False + _cartopy_enabled = False else: - _CARTOPY_ENABLED = True + _cartopy_enabled = True try: from mpl_toolkits.basemap import Basemap except ImportError: - _BASEMAP_ENABLED = False + _basemap_enabled = False else: - _BASEMAP_ENABLED = True + _basemap_enabled = True try: from Ngl import Resources except ImportError: - _PYNGL_ENABLED = False + _pyngl_enabled = False else: - _PYNGL_ENABLED = True + _pyngl_enabled = True + +_cache_size = 5 def xarray_enabled(): - global _XARRAY_ENABLED - return _XARRAY_ENABLED + global _xarray_enabled + return _xarray_enabled def disable_xarray(): - global _XARRAY_ENABLED - _XARRAY_ENABLED = False + global _xarray_enabled + _xarray_enabled = False def enable_xarray(): - global _XARRAY_ENABLED - _XARRAY_ENABLED = True + global _xarray_enabled + _xarray_enabled = True def cartopy_enabled(): - global _CARTOPY_ENABLED - return _CARTOPY_ENABLED + global _cartopy_enabled + return _cartopy_enabled def enable_cartopy(): - global _CARTOPY_ENABLED - _CARTOPY_ENABLED = True + global _cartopy_enabled + _cartopy_enabled = True def disable_cartopy(): - global _CARTOPY_ENABLED - _CARTOPY_ENABLED = True + global _cartopy_enabled + _cartopy_enabled = True def basemap_enabled(): - global _BASEMAP_ENABLED - return _BASEMAP_ENABLED + global _basemap_enabled + return _basemap_enabled def enable_basemap(): - global _BASEMAP_ENABLED - _BASEMAP_ENABLED = True + global _basemap_enabled + _basemap_enabled = True def disable_basemap(): - global _BASEMAP_ENABLED - _BASEMAP_ENABLED = True + global _basemap_enabled + _basemap_enabled = True def pyngl_enabled(): - global _PYNGL_ENABLED - return _PYNGL_ENABLED + global _pyngl_enabled + return _pyngl_enabled def enable_pyngl(): - global _PYNGL_ENABLED - _PYNGL_ENABLED = True + global _pyngl_enabled + _pyngl_enabled = True def disable_pyngl(): - global _PYNGL_ENABLED - _PYNGL_ENABLED = True + global _pyngl_enabled + _pyngl_enabled = True + +def set_cache_size(size): + global _cache_size + _cache_size = size + +def get_cache_size(): + return int(_cache_size) + + + diff --git a/src/wrf/metadecorators.py b/src/wrf/metadecorators.py index 4062aaf..5abc508 100644 --- a/src/wrf/metadecorators.py +++ b/src/wrf/metadecorators.py @@ -369,8 +369,8 @@ def set_cloudfrac_metadata(): outdimnames[-2:] = copy_var.dims[-2:] # Left dims outdimnames[1:-2] = copy_var.dims[0:-3] - outdimnames[0] = "low_med_high" - outattrs["description"] = "low, med, high clouds" + outdimnames[0] = "low_mid_high" + outattrs["description"] = "low, mid, high clouds" outattrs["MemoryOrder"] = "XY" outattrs["units"] = "%" outname = "cloudfrac" @@ -388,7 +388,7 @@ def set_cloudfrac_metadata(): elif key == "Time": outcoords[key] = npvalues(dataarray) - outcoords["low_med_high"] = ["low", "med", "high"] + outcoords["low_mid_high"] = ["low", "mid", "high"] return DataArray(result, name=outname, coords=outcoords, dims=outdimnames, attrs=outattrs) @@ -1342,7 +1342,7 @@ def set_cloudfrac_alg_metadata(copyarg="pres"): outattrs = OrderedDict() outname = "cloudfrac" - outattrs["description"] = "low, med, high clouds" + outattrs["description"] = "low, mid, high clouds" outattrs["units"] = "%" outattrs["MemoryOrder"] = "XY" @@ -1357,9 +1357,9 @@ def set_cloudfrac_alg_metadata(copyarg="pres"): outcoords = {} - # Left-most is always cape_cin or cape_cin_lcl_lfc - outdims[0] = "low_med_high" - outcoords["low_med_high"] = ["low", "med", "high"] + # Left-most is always low_mid_high + outdims[0] = "low_mid_high" + outcoords["low_mid_high"] = ["low", "mid", "high"] out = DataArray(result, name=outname, dims=outdims, coords=outcoords, attrs=outattrs) diff --git a/src/wrf/routines.py b/src/wrf/routines.py index 598eb5b..95ec676 100644 --- a/src/wrf/routines.py +++ b/src/wrf/routines.py @@ -129,7 +129,8 @@ _ALIASES = {"cape_2d" : "cape2d", "ter" : "terrain", "updraft_helicity" : "uhel", "td" : "dp", - "td2" : "dp2m" + "td2" : "dp2m", + "cfrac" : "cloudfrac" } class ArgumentError(Exception): diff --git a/src/wrf/util.py b/src/wrf/util.py index 4e3eca5..ffa5009 100644 --- a/src/wrf/util.py +++ b/src/wrf/util.py @@ -325,18 +325,20 @@ def is_moving_domain(wrfseq, varname=None, latvar=either("XLAT", "XLAT_M"), # Quick check on pressure coordinates, bypassing the need to search the # domain corner points - try: - coord_names = getattr(first_wrfnc.variables["P"], - "coordinates").split() - except KeyError: - pass - else: - if "XTIME" in coord_names: - return True - else: - return False +# try: +# coord_names = getattr(first_wrfnc.variables["P"], +# "coordinates").split() +# except KeyError: +# pass +# else: +# if "XTIME" in coord_names: +# return True +# else: +# return False # The long way of checking all lat/lon corner points + # There doesn't appear to be a shortcut, so this should probably + # be stored in a cache somewhere if varname is not None: try: coord_names = getattr(first_wrfnc.variables[varname], @@ -349,18 +351,18 @@ def is_moving_domain(wrfseq, varname=None, latvar=either("XLAT", "XLAT_M"), else: # If the XTIME variable is found to be a coordinate variable, # then it's a moving domain file - try: - xtime_coord = coord_names[2] - except IndexError: - # XTIME is not a coordinate variable, if the variable is in the - # file, then this is not a moving domain file - if "XTIME" in first_wrfnc.variables: - return False - - else: - # XTIME is a coordinate, so this is a moving domain file - if xtime_coord == "XTIME": - return True +# try: +# xtime_coord = coord_names[2] +# except IndexError: +# # XTIME is not a coordinate variable, if the variable is in the +# # file, then this is not a moving domain file +# if "XTIME" in first_wrfnc.variables: +# return False +# +# else: +# # XTIME is a coordinate, so this is a moving domain file +# if xtime_coord == "XTIME": +# return True lon_coord = coord_names[0] lat_coord = coord_names[1] @@ -395,8 +397,9 @@ def is_moving_domain(wrfseq, varname=None, latvar=either("XLAT", "XLAT_M"), else: if _corners_moved(wrfnc, ll_corner, ur_corner, lat_coord, lon_coord): + return True - + return False def _get_global_attr(wrfnc, attr): diff --git a/test/cachetest.py b/test/cachetest.py new file mode 100644 index 0000000..cc7efee --- /dev/null +++ b/test/cachetest.py @@ -0,0 +1,65 @@ +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +from threading import Thread +from Queue import Queue +from collections import OrderedDict + +import unittest as ut +import numpy.testing as nt + +from wrf.cache import cache_item, get_cached_item, _get_cache +from wrf.config import get_cache_size + + +class TestThread(Thread): + def __init__(self, num, q): + self.num = num + self.q = q + super(TestThread, self).__init__() + + def run(self): + for i in range(get_cache_size() + 10): + key = "A" + str(i) + cache_item(key, "test", i * self.num) + + item = get_cached_item(key, "test") + + if item != i * self.num: + raise RuntimeError("cache is bogus") + + cache = OrderedDict(_get_cache()) + + self.q.put(cache) + + +class CacheTest(ut.TestCase): + longMessage = True + + def test_thread_local(self): + q1 = Queue() + q2 = Queue() + thread1 = TestThread(2, q1) + thread2 = TestThread(40, q2) + + thread1.start() + thread2.start() + + result1 = q1.get(True, 1) + result2 = q2.get(True, 1) + + thread1.join() + thread2.join() + + print(result1) + print(result2) + + # Result 1 and 2 shoudl be different + self.assertNotEqual(result1, result2) + + # This thread should have no cache + self.assertIsNone(_get_cache()) + + +if __name__ == "__main__": + ut.main() \ No newline at end of file diff --git a/test/ncl_get_var.ncl b/test/ncl_get_var.ncl index 17a38e4..45ba948 100644 --- a/test/ncl_get_var.ncl +++ b/test/ncl_get_var.ncl @@ -22,7 +22,7 @@ "geopt", "helicity", "lat", "lon", "omg", "p", "pressure", \ "pvo", "pw", "rh2", "rh", "slp", "ter", "td2", "td", "tc", \ "theta", "tk", "tv", "twb", "updraft_helicity", "ua", "va", \ - "wa", "uvmet10", "uvmet", "z"/] + "wa", "uvmet10", "uvmet", "z", "cfrac"/] unique_dimname_list = NewList("fifo") unique_dimsize_list = NewList("fifo") diff --git a/test/utests.py b/test/utests.py index 03c3efe..aae3433 100644 --- a/test/utests.py +++ b/test/utests.py @@ -82,16 +82,15 @@ def make_test(varname, wrf_in, referent, multi=False, repeat=3, pynio=False): wrf_file = wrf_in nc = Nio.open_file(wrf_file) in_wrfnc = [nc for i in xrange(repeat)] - - # Note: remove this after cloudfrac is included in NCL - # For now just make sure it runs - if varname == "cloudfrac": - my_vals = getvar(in_wrfnc, "cloudfrac", timeidx=timeidx) - return refnc = NetCDF(referent) + # These have a left index that defines the product type + multiproduct = varname in ("uvmet", "uvmet10", "cape_2d", "cape_3d", + "cfrac") + + if not multi: ref_vals = refnc.variables[varname][:] else: @@ -104,6 +103,8 @@ def make_test(varname, wrf_in, referent, multi=False, repeat=3, pynio=False): new_dims = [2] + [repeat] + [x for x in data.shape[1:]] elif (varname == "cape_2d"): new_dims = [4] + [repeat] + [x for x in data.shape[1:]] + elif (varname == "cfrac"): + new_dims = [3] + [repeat] + [x for x in data.shape[1:]] masked=False @@ -114,34 +115,20 @@ def make_test(varname, wrf_in, referent, multi=False, repeat=3, pynio=False): ref_vals = np.zeros(new_dims, data.dtype) else: ref_vals = ma.asarray(np.zeros(new_dims, data.dtype)) - + for i in xrange(repeat): - if (varname != "uvmet" and varname != "uvmet10" - and varname != "cape_2d" and varname != "cape_3d"): + if not multiproduct: ref_vals[i,:] = data[:] if masked: ref_vals.mask[i,:] = data.mask[:] - elif (varname == "uvmet" or varname == "uvmet10" - or varname=="cape_3d"): - ref_vals[0, i, :] = data[0,:] - ref_vals[1, i, :] = data[1,:] - - if masked: - ref_vals.mask[0,i,:] = data.mask[0,:] - ref_vals.mask[1,i,:] = data.mask[1,:] - elif varname == "cape_2d": - ref_vals[0, i, :] = data[0,:] - ref_vals[1, i, :] = data[1,:] - ref_vals[2, i, :] = data[2,:] - ref_vals[3, i, :] = data[3,:] - - if masked: - ref_vals.mask[0,i,:] = data.mask[0,:] - ref_vals.mask[1,i,:] = data.mask[1,:] - ref_vals.mask[2,i,:] = data.mask[2,:] - ref_vals.mask[3,i,:] = data.mask[3,:] - + + else: + for prod in xrange(ref_vals.shape[0]): + ref_vals[prod,i,:] = data[prod,:] + + if masked: + ref_vals.mask[prod,i,:] = data.mask[prod,:] if (varname == "tc"): my_vals = getvar(in_wrfnc, "temp", timeidx=timeidx, units="c") @@ -483,7 +470,7 @@ if __name__ == "__main__": "geopt", "helicity", "lat", "lon", "omg", "p", "pressure", "pvo", "pw", "rh2", "rh", "slp", "ter", "td2", "td", "tc", "theta", "tk", "tv", "twb", "updraft_helicity", "ua", "va", - "wa", "uvmet10", "uvmet", "z", "cloudfrac"] + "wa", "uvmet10", "uvmet", "z", "cfrac"] interp_methods = ["interplevel", "vertcross", "interpline", "vinterp"] try: