forked from 3rdparty/wrf-python
Bill Ladwig
9 years ago
14 changed files with 521 additions and 157 deletions
@ -0,0 +1,183 @@
@@ -0,0 +1,183 @@
|
||||
''' |
||||
Created on Jan 16, 2014 |
||||
|
||||
@author: sean |
||||
''' |
||||
from __future__ import absolute_import, division, print_function |
||||
|
||||
from functools import partial |
||||
import json |
||||
import logging |
||||
import os |
||||
import sys |
||||
|
||||
import jinja2 |
||||
|
||||
from .conda_interface import PY3 |
||||
from .environ import get_dict as get_environ |
||||
from .metadata import select_lines, ns_cfg |
||||
from .source import WORK_DIR |
||||
|
||||
log = logging.getLogger(__file__) |
||||
|
||||
|
||||
class UndefinedNeverFail(jinja2.Undefined): |
||||
""" |
||||
A class for Undefined jinja variables. |
||||
This is even less strict than the default jinja2.Undefined class, |
||||
because it permits things like {{ MY_UNDEFINED_VAR[:2] }} and |
||||
{{ MY_UNDEFINED_VAR|int }}. This can mask lots of errors in jinja templates, so it |
||||
should only be used for a first-pass parse, when you plan on running a 'strict' |
||||
second pass later. |
||||
""" |
||||
all_undefined_names = [] |
||||
|
||||
def __init__(self, hint=None, obj=jinja2.runtime.missing, name=None, |
||||
exc=jinja2.exceptions.UndefinedError): |
||||
UndefinedNeverFail.all_undefined_names.append(name) |
||||
jinja2.Undefined.__init__(self, hint, obj, name, exc) |
||||
|
||||
__add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \ |
||||
__truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \ |
||||
__mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ |
||||
__getitem__ = __lt__ = __le__ = __gt__ = __ge__ = \ |
||||
__complex__ = __pow__ = __rpow__ = \ |
||||
lambda self, *args, **kwargs: UndefinedNeverFail(hint=self._undefined_hint, |
||||
obj=self._undefined_obj, |
||||
name=self._undefined_name, |
||||
exc=self._undefined_exception) |
||||
|
||||
__str__ = __repr__ = \ |
||||
lambda *args, **kwargs: u'' |
||||
|
||||
__int__ = lambda _: 0 |
||||
__float__ = lambda _: 0.0 |
||||
|
||||
def __getattr__(self, k): |
||||
try: |
||||
return object.__getattr__(self, k) |
||||
except AttributeError: |
||||
return UndefinedNeverFail(hint=self._undefined_hint, |
||||
obj=self._undefined_obj, |
||||
name=self._undefined_name + '.' + k, |
||||
exc=self._undefined_exception) |
||||
|
||||
|
||||
class FilteredLoader(jinja2.BaseLoader): |
||||
""" |
||||
A pass-through for the given loader, except that the loaded source is |
||||
filtered according to any metadata selectors in the source text. |
||||
""" |
||||
|
||||
def __init__(self, unfiltered_loader): |
||||
self._unfiltered_loader = unfiltered_loader |
||||
self.list_templates = unfiltered_loader.list_templates |
||||
|
||||
def get_source(self, environment, template): |
||||
contents, filename, uptodate = self._unfiltered_loader.get_source(environment, |
||||
template) |
||||
return select_lines(contents, ns_cfg()), filename, uptodate |
||||
|
||||
|
||||
def load_setup_py_data(setup_file='setup.py', from_recipe_dir=False, recipe_dir=None, |
||||
unload_modules=None, fail_on_error=False): |
||||
|
||||
_setuptools_data = {} |
||||
|
||||
def setup(**kw): |
||||
_setuptools_data.update(kw) |
||||
|
||||
import setuptools |
||||
import distutils.core |
||||
|
||||
try: |
||||
import numpy.distutils.core |
||||
except ImportError: |
||||
do_numpy = False |
||||
else: |
||||
do_numpy = True |
||||
|
||||
cd_to_work = False |
||||
|
||||
if from_recipe_dir and recipe_dir: |
||||
setup_file = os.path.abspath(os.path.join(recipe_dir, setup_file)) |
||||
elif os.path.exists(WORK_DIR): |
||||
cd_to_work = True |
||||
cwd = os.getcwd() |
||||
os.chdir(WORK_DIR) |
||||
if not os.path.isabs(setup_file): |
||||
setup_file = os.path.join(WORK_DIR, setup_file) |
||||
# this is very important - or else if versioneer or otherwise is in the start folder, |
||||
# things will pick up the wrong versioneer/whatever! |
||||
sys.path.insert(0, WORK_DIR) |
||||
else: |
||||
log.debug("Did not find setup.py file in manually specified location, and source " |
||||
"not downloaded yet.") |
||||
return {} |
||||
|
||||
# Patch setuptools, distutils |
||||
setuptools_setup = setuptools.setup |
||||
distutils_setup = distutils.core.setup |
||||
setuptools.setup = distutils.core.setup = setup |
||||
|
||||
if do_numpy: |
||||
numpy_setup = numpy.distutils.core.setup |
||||
numpy.distutils.core.setup = setup |
||||
|
||||
ns = { |
||||
'__name__': '__main__', |
||||
'__doc__': None, |
||||
'__file__': setup_file, |
||||
} |
||||
try: |
||||
code = compile(open(setup_file).read(), setup_file, 'exec', dont_inherit=1) |
||||
exec(code, ns, ns) |
||||
distutils.core.setup = distutils_setup |
||||
setuptools.setup = setuptools_setup |
||||
if do_numpy: |
||||
numpy.distutils.core.setup = numpy_setup |
||||
# this happens if setup.py is used in load_setup_py_data, but source is not yet downloaded |
||||
except: |
||||
raise |
||||
finally: |
||||
if cd_to_work: |
||||
os.chdir(cwd) |
||||
del sys.path[0] |
||||
|
||||
return _setuptools_data |
||||
|
||||
|
||||
def load_setuptools(setup_file='setup.py', from_recipe_dir=False, recipe_dir=None, |
||||
unload_modules=None, fail_on_error=False): |
||||
log.warn("Deprecation notice: the load_setuptools function has been renamed to " |
||||
"load_setup_py_data. load_setuptools will be removed in a future release.") |
||||
return load_setup_py_data(setup_file=setup_file, from_recipe_dir=from_recipe_dir, |
||||
recipe_dir=recipe_dir, unload_modules=unload_modules, |
||||
fail_on_error=fail_on_error) |
||||
|
||||
|
||||
def load_npm(): |
||||
# json module expects bytes in Python 2 and str in Python 3. |
||||
mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'} |
||||
with open('package.json', **mode_dict) as pkg: |
||||
return json.load(pkg) |
||||
|
||||
|
||||
def context_processor(initial_metadata, recipe_dir): |
||||
""" |
||||
Return a dictionary to use as context for jinja templates. |
||||
|
||||
initial_metadata: Augment the context with values from this MetaData object. |
||||
Used to bootstrap metadata contents via multiple parsing passes. |
||||
""" |
||||
ctx = get_environ(m=initial_metadata) |
||||
environ = dict(os.environ) |
||||
environ.update(get_environ(m=initial_metadata)) |
||||
|
||||
ctx.update( |
||||
load_setup_py_data=partial(load_setup_py_data, recipe_dir=recipe_dir), |
||||
# maintain old alias for backwards compatibility: |
||||
load_setuptools=partial(load_setuptools, recipe_dir=recipe_dir), |
||||
load_npm=load_npm, |
||||
environ=environ) |
||||
return ctx |
||||
@ -0,0 +1,52 @@
@@ -0,0 +1,52 @@
|
||||
from __future__ import (absolute_import, division, print_function, |
||||
unicode_literals) |
||||
|
||||
from threading import local |
||||
from collections import OrderedDict |
||||
|
||||
from .config import get_cache_size |
||||
|
||||
_local_storage = local() |
||||
|
||||
def cache_item(key, product, value): |
||||
global _local_storage |
||||
|
||||
try: |
||||
cache = _local_storage.cache |
||||
except AttributeError: |
||||
_local_storage.cache = OrderedDict() |
||||
cache = _local_storage.cache |
||||
|
||||
try: |
||||
prod_dict = cache[key] |
||||
except KeyError: |
||||
if len(cache) >= get_cache_size(): |
||||
cache.popitem(last=False) # Remove the oldest dataset |
||||
|
||||
cache[key] = OrderedDict() |
||||
prod_dict = cache[key] |
||||
|
||||
cache[key][product] = value |
||||
|
||||
|
||||
def get_cached_item(key, product): |
||||
cache = getattr(_local_storage, "cache", None) |
||||
|
||||
if cache is None: |
||||
return None |
||||
|
||||
prod_dict = cache.get(key, None) |
||||
|
||||
if prod_dict is None: |
||||
return None |
||||
|
||||
return prod_dict.get(product, None) |
||||
|
||||
|
||||
def _get_cache(): |
||||
return getattr(_local_storage, "cache", None) |
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,65 @@
@@ -0,0 +1,65 @@
|
||||
from __future__ import (absolute_import, division, print_function, |
||||
unicode_literals) |
||||
|
||||
from threading import Thread |
||||
from Queue import Queue |
||||
from collections import OrderedDict |
||||
|
||||
import unittest as ut |
||||
import numpy.testing as nt |
||||
|
||||
from wrf.cache import cache_item, get_cached_item, _get_cache |
||||
from wrf.config import get_cache_size |
||||
|
||||
|
||||
class TestThread(Thread): |
||||
def __init__(self, num, q): |
||||
self.num = num |
||||
self.q = q |
||||
super(TestThread, self).__init__() |
||||
|
||||
def run(self): |
||||
for i in range(get_cache_size() + 10): |
||||
key = "A" + str(i) |
||||
cache_item(key, "test", i * self.num) |
||||
|
||||
item = get_cached_item(key, "test") |
||||
|
||||
if item != i * self.num: |
||||
raise RuntimeError("cache is bogus") |
||||
|
||||
cache = OrderedDict(_get_cache()) |
||||
|
||||
self.q.put(cache) |
||||
|
||||
|
||||
class CacheTest(ut.TestCase): |
||||
longMessage = True |
||||
|
||||
def test_thread_local(self): |
||||
q1 = Queue() |
||||
q2 = Queue() |
||||
thread1 = TestThread(2, q1) |
||||
thread2 = TestThread(40, q2) |
||||
|
||||
thread1.start() |
||||
thread2.start() |
||||
|
||||
result1 = q1.get(True, 1) |
||||
result2 = q2.get(True, 1) |
||||
|
||||
thread1.join() |
||||
thread2.join() |
||||
|
||||
print(result1) |
||||
print(result2) |
||||
|
||||
# Result 1 and 2 shoudl be different |
||||
self.assertNotEqual(result1, result2) |
||||
|
||||
# This thread should have no cache |
||||
self.assertIsNone(_get_cache()) |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
ut.main() |
||||
Loading…
Reference in new issue