Cleaned up cape fortran routine. Fixed issue with generators not being copied correctly due to changes in python (the python works as expected now). Fixed a uniqueness problem with coordinate caching which was causing problems in jupyter notebook when files were changed. Fixed an issue with the cache test script failing due to unitialized thread local data in child threads. Fixes #34. Fixes #14.

fortran/eqthecalc.f90

@@ -32,8 +32,7 @@ SUBROUTINE DEQTHECALC(qvp, tmk, prs, eth, miy, mjx, mkzh)
     REAL(KIND=8) :: tlcl
     INTEGER :: i, j, k
 
-    ! Note:  removing temporaries did not improve performance for this algorithm.
+    !$OMP PARALLEL DO COLLAPSE(3) PRIVATE(i, j, k, q, t, p, e, tlcl)
-    !$OMP PARALLEL DO COLLAPSE(3) PRIVATE(i,j,k,q,t,p,e,tlcl)
     DO k = 1,mkzh
         DO j = 1,mjx
             DO i = 1,miy

src/wrf/config.py

@@ -2,35 +2,64 @@ from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
 
 from threading import local
+import wrapt
 
 _local_config = local()
-_local_config.xarray_enabled = True
-_local_config.cartopy_enabled = True
-_local_config.basemap_enabled = True
-_local_config.pyngl_enabled = True
-_local_config.cache_size = 20
-
-try:
-    from xarray import DataArray
-except ImportError:
-    _local_config.xarray_enabled = False
 
-try:
+def _init_local():
-    from cartopy import crs
+    global _local_config
-except ImportError:
+    
-    _local_config.cartopy_enabled = False
+    _local_config.xarray_enabled = True
+    _local_config.cartopy_enabled = True
+    _local_config.basemap_enabled = True
+    _local_config.pyngl_enabled = True
+    _local_config.cache_size = 20
+    _local_config.initialized = True
+    
+    try:
+        from xarray import DataArray
+    except ImportError:
+        _local_config.xarray_enabled = False
         
-try:
+    try:
-    from mpl_toolkits.basemap import Basemap
+        from cartopy import crs
-except ImportError:
+    except ImportError:
-    _local_config.basemap_enabled = False
+        _local_config.cartopy_enabled = False
         
-try:
+    try:
-    from Ngl import Resources
+        from mpl_toolkits.basemap import Basemap
-except ImportError:
+    except ImportError:
-    _local_config.pyngl_enabled = False
+        _local_config.basemap_enabled = False
         
+    try:
+        from Ngl import Resources
+    except ImportError:
+        _local_config.pyngl_enabled = False
         
+
+# Initialize the main thread's configuration
+_init_local()
+
+
+def init_local():
+    """A decorator that initializes thread local data if necessary."""
+    @wrapt.decorator
+    def func_wrapper(wrapped, instance, args, kwargs):
+        global _local_config
+        try:
+            init = _local_config.init
+        except AttributeError:
+            _init_local()
+        else:
+            if not init:
+                _init_local() 
+                
+        return wrapped(*args, **kwargs)
+        
+    return func_wrapper
+
+
+@init_local()
 def xarray_enabled():
     """Return True if xarray is installed and enabled.
     
@@ -43,18 +72,21 @@ def xarray_enabled():
     return _local_config.xarray_enabled
 
 
+@init_local()
 def disable_xarray():
     """Disable xarray."""
     global _local_config
     _local_config.xarray_enabled = False
     
 
+@init_local()
 def enable_xarray():
     """Enable xarray."""
     global _local_config
     _local_config.xarray_enabled = True
     
 
+@init_local()
 def cartopy_enabled():
     """Return True if cartopy is installed and enabled.
     
@@ -67,18 +99,21 @@ def cartopy_enabled():
     return _local_config.cartopy_enabled
 
 
+@init_local()
 def enable_cartopy():
     """Enable cartopy."""
     global _local_config
     _local_config.cartopy_enabled = True
     
 
+@init_local()
 def disable_cartopy():
     """Disable cartopy."""
     global _local_config
     _local_config.cartopy_enabled = True
     
 
+@init_local()
 def basemap_enabled():
     """Return True if basemap is installed and enabled.
     
@@ -91,17 +126,21 @@ def basemap_enabled():
     return _local_config.basemap_enabled
 
 
+@init_local()
 def enable_basemap():
     """Enable basemap."""
     global _local_config
     _local_config.basemap_enabled = True
     
 
+@init_local()
 def disable_basemap():
     """Disable basemap."""
     global _local_config
     _local_config.basemap_enabled = True
 
+
+@init_local()
 def pyngl_enabled():
     """Return True if pyngl is installed and enabled.
     
@@ -114,18 +153,21 @@ def pyngl_enabled():
     return _local_config.pyngl_enabled
 
 
+@init_local()
 def enable_pyngl():
     """Enable pyngl."""
     global _local_config
     _local_config.pyngl_enabled = True
     
 
+@init_local()  
 def disable_pyngl():
     """Disable pyngl."""
     global _local_config
     _local_config.pyngl_enabled = True
     
 
+@init_local() 
 def set_cache_size(size):
     """Set the maximum number of items that the threadlocal cache can retain.
     
@@ -144,6 +186,7 @@ def set_cache_size(size):
     _local_config.cache_size = size
     
 
+@init_local()  
 def get_cache_size():
     """Return the maximum number of items that the threadlocal cache can retain.
     

src/wrf/util.py

@@ -209,11 +209,30 @@ def _generator_copy(gen):
     module = getmodule(gen.gi_frame)
     
     if module is not None:
-        res = module.get(funcname)(**argvals.locals)
+        try:
+            try:
+                argd = {key:argvals.locals[key] for key in argvals.args}
+                res = module.get(funcname)(**argd)
+            except AttributeError:
+                res = getattr(module, funcname)(**argd)
+        except:
+            # This is the old way it used to work, but it looks like this was
+            # fixed by Python.  
+            try:
+                res = module.get(funcname)(**argvals.locals)
+            except AttributeError:
+                res = getattr(module, funcname)(**argvals.locals)    
     else:
         # Created in jupyter or the python interpreter
         import __main__
-        res = getattr(__main__, funcname)(**argvals.locals)
+        
+        try:
+            argd = {key:argvals.locals[key] for key in argvals.args}
+            res = getattr(__main__, funcname)(**argd)
+        except:
+            # This was the old way it used to work, but appears to have 
+            # been fixed by Python.
+            res = getattr(__main__, funcname)(**argvals.locals)
         
     return res
 
@@ -2583,26 +2602,6 @@ def get_proj_params(wrfin):#, timeidx=0, varname=None):
                                                 "DX", "DY"))
     
     return proj_params
-#     multitime = is_multi_time_req(timeidx)
-#     if not multitime:
-#         time_idx_or_slice = timeidx
-#     else:
-#         time_idx_or_slice = slice(None)
-#     
-#     if varname is not None:
-#         if not is_coordvar(varname):
-#             coord_names = getattr(wrfin.variables[varname], 
-#                                   "coordinates").split()
-#             lon_coord = coord_names[0]
-#             lat_coord = coord_names[1]
-#         else:
-#             lat_coord, lon_coord = get_coord_pairs(varname)
-#     else:
-#         lat_coord, lon_coord = latlon_coordvars(wrfin.variables)
-#     
-#     return (wrfin.variables[lat_coord][time_idx_or_slice,:],
-#             wrfin.variables[lon_coord][time_idx_or_slice,:],
-#             proj_params)
         
 
 def from_args(func, argnames, *args, **kwargs):
@@ -2936,18 +2935,31 @@ def psafilepath():
     return os.path.join(os.path.dirname(__file__), "data", "psadilookup.dat")
 
 
-def get_id(obj):
+def get_filepath(obj):
-    """Return the object id.
     
-    The object id is used as a caching key for various routines.  If the
+    try:
+        path = obj.filepath()
+    except AttributeError:
+        try:
+            path = obj.file.path
+        except:
+            raise ValueError("file contains no path information")
+    
+    return path
+
+def get_id(obj, prefix=''):
+    """Return the cache id.
+    
+    The cache id is used as a caching key for various routines. If the
     object type is a mapping, then the result will also be a 
-    mapping of each key to the object id for the value.  Otherwise, only the 
+    mapping of each key to the object id for the value.
-    object id is returned.
     
     Args:
     
         obj (:obj:`object`): Any object type.
         
+        prefix (:obj:`str`): A string to help with recursive calls.
+        
     Returns:
     
         :obj:`int` or :obj:`dict`: If the *obj* parameter is not a mapping, 
@@ -2955,12 +2967,18 @@ def get_id(obj):
         key to the object id for the value is returned.
     
     """
+    if not is_multi_file(obj):
+        return hash(prefix + get_filepath(obj))
+    
+    # For sequences, the hashing string will be the list ID and the 
+    # path for the first file in the sequence
     if not is_mapping(obj):
-        return id(obj)
+        _next = next(iter(obj))
+        return get_id(_next, prefix + str(id(obj)))
     
     # For each key in the mapping, recursively call get_id until
     # until a non-mapping is found
-    return {key : get_id(val) for key,val in viewitems(obj)}
+    return {key : get_id(val, prefix) for key,val in viewitems(obj)}
 
 
 def geo_bounds(var=None, wrfin=None, varname=None, timeidx=0, method="cat",

test/cachetest.py

@@ -2,7 +2,10 @@ from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
 
 from threading import Thread
-from Queue import Queue
+try:
+    from Queue import Queue
+except ImportError:
+    from queue import Queue
 from collections import OrderedDict
 
 import unittest as ut

test/generator_test.py

@@ -0,0 +1,17 @@
+from __future__ import (absolute_import, division, print_function, unicode_literals)
+
+from wrf import getvar
+from netCDF4 import Dataset as nc
+#ncfile = nc("/Users/ladwig/Documents/wrf_files/wrfout_d01_2016-02-25_18_00_00")
+ncfile = nc("/Users/ladwig/Documents/wrf_files/wrfout_d01_2016-10-07_00_00_00")
+
+def gen_seq():
+    wrfseq = [ncfile, ncfile, ncfile]
+    for wrf in wrfseq:
+        yield wrf
+        
+p_gen = getvar(gen_seq(), "P", method="join")
+
+print(p_gen)
+del p_gen
+