From f62f6b96a594c3800b0cddf0228c4ff81fe4fb96 Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Fri, 19 Aug 2022 10:07:18 -0400 Subject: [PATCH] core - sqlkv cache file (#7659) --- c7n/cache.py | 137 ++++++++++++------- c7n/policy.py | 3 +- c7n/query.py | 3 + c7n/resources/iam.py | 33 +++-- poetry.lock | 35 +++-- pyproject.toml | 1 + tests/test_cache.py | 193 ++++++++++----------------- tests/test_iam.py | 9 +- tools/c7n_azure/c7n_azure/filters.py | 1 - tools/c7n_azure/c7n_azure/query.py | 3 +- tools/c7n_gcp/c7n_gcp/query.py | 26 +++- 11 files changed, 227 insertions(+), 217 deletions(-) diff --git a/c7n/cache.py b/c7n/cache.py index aa75e0a2f51..8da1e80aa71 100644 --- a/c7n/cache.py +++ b/c7n/cache.py @@ -5,9 +5,10 @@ """ import pickle # nosec nosemgrep +from datetime import datetime, timedelta import os import logging -import time +import sqlite3 log = logging.getLogger('custodian.cache') @@ -30,12 +31,11 @@ def factory(config): if not CACHE_NOTIFY: log.debug("Using in-memory cache") CACHE_NOTIFY = True - return InMemoryCache() + return InMemoryCache(config) + return SqlKvCache(config) - return FileCacheManager(config) - -class NullCache: +class Cache: def __init__(self, config): self.config = config @@ -52,74 +52,115 @@ def save(self, key, data): def size(self): return 0 + def close(self): + pass + + def __enter__(self): + self.load() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + -class InMemoryCache: +class NullCache(Cache): + pass + + +class InMemoryCache(Cache): # Running in a temporary environment, so keep as a cache. __shared_state = {} - def __init__(self): + def __init__(self, config): + super().__init__(config) self.data = self.__shared_state def load(self): return True def get(self, key): - return self.data.get(pickle.dumps(key)) # nosemgrep + return self.data.get(encode(key)) def save(self, key, data): - self.data[pickle.dumps(key)] = data # nosemgrep + self.data[encode(key)] = data def size(self): return sum(map(len, self.data.values())) -class FileCacheManager: +def encode(key): + return pickle.dumps(key, protocol=pickle.HIGHEST_PROTOCOL) # nosemgrep + + +def resolve_path(path): + return os.path.abspath( + os.path.expanduser( + os.path.expandvars(path))) + + +class SqlKvCache(Cache): + + create_table = """ + create table if not exists c7n_cache ( + key blob primary key, + value blob, + create_date timestamp + ) + """ def __init__(self, config): - self.config = config + super().__init__(config) self.cache_period = config.cache_period - self.cache_path = os.path.abspath( - os.path.expanduser( - os.path.expandvars( - config.cache))) - self.data = {} - - def get(self, key): - k = pickle.dumps(key) # nosemgrep - return self.data.get(k) + self.cache_path = resolve_path(config.cache) + self.conn = None def load(self): - if self.data: - return True - if os.path.isfile(self.cache_path): - if (time.time() - os.stat(self.cache_path).st_mtime > - self.config.cache_period * 60): - return False + # migration from pickle cache file + if os.path.exists(self.cache_path): with open(self.cache_path, 'rb') as fh: - try: - self.data = pickle.load(fh) # nosec nosemgrep - except EOFError: - return False - log.debug("Using cache file %s" % self.cache_path) - return True + header = fh.read(15) + if header != b'SQLite format 3': + log.debug('removing old cache file') + os.remove(self.cache_path) + elif not os.path.exists(os.path.dirname(self.cache_path)): + # parent directory creation + os.makedirs(os.path.dirname(self.cache_path)) + self.conn = sqlite3.connect(self.cache_path) + self.conn.execute(self.create_table) + with self.conn as cursor: + log.debug('expiring stale cache entries') + cursor.execute( + 'delete from c7n_cache where create_date < ?', + [datetime.utcnow() - timedelta(minutes=self.cache_period)]) + return True - def save(self, key, data): - try: - with open(self.cache_path, 'wb') as fh: # nosec - self.data[pickle.dumps(key)] = data # nosemgrep - pickle.dump(self.data, fh, protocol=2) # nosemgrep - except Exception as e: - log.warning("Could not save cache %s err: %s" % ( - self.cache_path, e)) - if not os.path.exists(self.cache_path): - directory = os.path.dirname(self.cache_path) - log.info('Generating Cache directory: %s.' % directory) - try: - os.makedirs(directory) - except Exception as e: - log.warning("Could not create directory: %s err: %s" % ( - directory, e)) + def get(self, key): + with self.conn as cursor: + r = cursor.execute( + 'select value, create_date from c7n_cache where key = ?', + [sqlite3.Binary(encode(key))] + ) + row = r.fetchone() + if row is None: + return None + value, create_date = row + create_date = sqlite3.converters['TIMESTAMP'](create_date.encode('utf8')) + if (datetime.utcnow() - create_date).total_seconds() / 60.0 > self.cache_period: + return None + return pickle.loads(value) # nosec nosemgrep + + def save(self, key, data, timestamp=None): + with self.conn as cursor: + timestamp = timestamp or datetime.utcnow() + cursor.execute( + 'replace into c7n_cache (key, value, create_date) values (?, ?, ?)', + (sqlite3.Binary(encode(key)), sqlite3.Binary(encode(data)), timestamp)) def size(self): return os.path.exists(self.cache_path) and os.path.getsize(self.cache_path) or 0 + + def close(self): + if self.conn: + self.conn.close() + self.conn = None diff --git a/c7n/policy.py b/c7n/policy.py index 3093925c029..fbeb8d9fe15 100644 --- a/c7n/policy.py +++ b/c7n/policy.py @@ -1285,8 +1285,7 @@ def __call__(self): resources = mode.provision() else: resources = mode.run() - # clear out resource manager post run, to clear cache - self.resource_manager = self.load_resource_manager() + return resources run = __call__ diff --git a/c7n/query.py b/c7n/query.py index d16e25c3875..da69c0166d5 100644 --- a/c7n/query.py +++ b/c7n/query.py @@ -528,6 +528,8 @@ def resources(self, query=None, augment=True) -> List[dict]: # Don't pollute cache with unaugmented resources. self._cache.save(cache_key, resources) + self._cache.close() + resource_count = len(resources) with self.ctx.tracer.subsegment('filter'): resources = self.filter_resources(resources) @@ -556,6 +558,7 @@ def _get_cached_resources(self, ids): m = self.get_model() id_set = set(ids) return [r for r in resources if r[m.id] in id_set] + self._cache.close() return None def get_resources(self, ids, cache=True, augment=True): diff --git a/c7n/resources/iam.py b/c7n/resources/iam.py index 41f1ee08e88..571528a2db8 100644 --- a/c7n/resources/iam.py +++ b/c7n/resources/iam.py @@ -1605,20 +1605,25 @@ def get_value_or_schema_default(self, k): return self.schema['properties'][k]['default'] def get_credential_report(self): - report = self.manager._cache.get('iam-credential-report') - if report: - return report - data = self.fetch_credential_report() - report = {} - if isinstance(data, bytes): - reader = csv.reader(io.StringIO(data.decode('utf-8'))) - else: - reader = csv.reader(io.StringIO(data)) - headers = next(reader) - for line in reader: - info = dict(zip(headers, line)) - report[info['user']] = self.process_user_record(info) - self.manager._cache.save('iam-credential-report', report) + cache = self.manager._cache + with cache: + cache_key = {'account': self.manager.config.account_id, 'iam-credential-report': True} + report = cache.get(cache_key) + + if report: + return report + data = self.fetch_credential_report() + report = {} + if isinstance(data, bytes): + reader = csv.reader(io.StringIO(data.decode('utf-8'))) + else: + reader = csv.reader(io.StringIO(data)) + headers = next(reader) + for line in reader: + info = dict(zip(headers, line)) + report[info['user']] = self.process_user_record(info) + cache.save(cache_key, report) + return report @classmethod diff --git a/poetry.lock b/poetry.lock index bd952b33df9..b5fde0ac0e3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -207,6 +207,17 @@ mccabe = ">=0.6.0,<0.7.0" pycodestyle = ">=2.7.0,<2.8.0" pyflakes = ">=2.3.0,<2.4.0" +[[package]] +name = "freezegun" +version = "1.2.2" +description = "Let your Python tests travel through time" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +python-dateutil = ">=2.7" + [[package]] name = "idna" version = "3.3" @@ -863,7 +874,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "49cb11b477adf87c04c9e56e4694aabf5afa53f1870ad23aa793dfc8bdd739c7" +content-hash = "79a1d8fb3eaae9c62d2125f77cd78d01fd668b8da0f2295c3f0705cb8fda9a8d" [metadata.files] argcomplete = [ @@ -876,20 +887,14 @@ aws-xray-sdk = [] bleach = [] boto3 = [] botocore = [] -certifi = [ - {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, - {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, -] +certifi = [] cffi = [] charset-normalizer = [] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] -colorama = [ - {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, - {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, -] +colorama = [] coverage = [ {file = "coverage-5.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf"}, {file = "coverage-5.5-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b"}, @@ -957,14 +962,12 @@ flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, ] +freezegun = [] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] -importlib-metadata = [ - {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, - {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, -] +importlib-metadata = [] importlib-resources = [] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -1257,11 +1260,7 @@ six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -tabulate = [ - {file = "tabulate-0.8.10-py3-none-any.whl", hash = "sha256:0ba055423dbaa164b9e456abe7920c5e8ed33fcc16f6d1b2f2d152c8e1e8b4fc"}, - {file = "tabulate-0.8.10-py3.8.egg", hash = "sha256:436f1c768b424654fce8597290d2764def1eea6a77cfa5c33be00b1bc0f4f63d"}, - {file = "tabulate-0.8.10.tar.gz", hash = "sha256:6c57f3f3dd7ac2782770155f3adb2db0b1a269637e42f27599925e64b114f519"}, -] +tabulate = [] termcolor = [ {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, ] diff --git a/pyproject.toml b/pyproject.toml index 1bc62c4f623..f83943f1d9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ psutil = "^5.7.0" twine = "^3.1.1" pytest-sugar = "^0.9.2" click = "^8.0" +freezegun = "^1.2.2" [tool.black] skip-string-normalization = true diff --git a/tests/test_cache.py b/tests/test_cache.py index e7524cb790a..3418d864fe7 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,12 +1,16 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 -from unittest import TestCase -from c7n import cache, config from argparse import Namespace -import pickle -import tempfile -import mock +from datetime import datetime, timedelta import os +import pickle +import sqlite3 +import sys +from unittest import TestCase + +import pytest + +from c7n import cache, config class TestCache(TestCase): @@ -14,7 +18,7 @@ class TestCache(TestCase): def test_factory(self): self.assertIsInstance(cache.factory(None), cache.NullCache) test_config = Namespace(cache_period=60, cache="test-cloud-custodian.cache") - self.assertIsInstance(cache.factory(test_config), cache.FileCacheManager) + self.assertIsInstance(cache.factory(test_config), cache.SqlKvCache) test_config.cache = None self.assertIsInstance(cache.factory(test_config), cache.NullCache) @@ -27,126 +31,71 @@ def test_mem_factory(self): cache.InMemoryCache) def test_get_set(self): - mem_cache = cache.InMemoryCache() + mem_cache = cache.InMemoryCache({}) mem_cache.save({'region': 'us-east-1'}, {'hello': 'world'}) self.assertEqual(mem_cache.size(), 1) self.assertEqual(mem_cache.load(), True) - mem_cache = cache.InMemoryCache() + mem_cache = cache.InMemoryCache({}) self.assertEqual( mem_cache.get({'region': 'us-east-1'}), {'hello': 'world'}) - - -class FileCacheManagerTest(TestCase): - - def setUp(self): - self.test_config = Namespace( - cache_period=60, cache="test-cloud-custodian.cache" - ) - self.test_cache = cache.FileCacheManager(self.test_config) - self.test_key = "test" - self.bad_key = "bad" - self.test_value = [1, 2, 3] - - def test_get_set(self): - t = self.temporary_file_with_cleanup() - c = cache.FileCacheManager(Namespace(cache_period=60, cache=t.name)) - self.assertFalse(c.load()) - k1 = {"account": "12345678901234", "region": "us-west-2", "resource": "ec2"} - c.save(k1, range(5)) - self.assertEqual(c.get(k1), range(5)) - k2 = {"account": "98765432101234", "region": "eu-west-1", "resource": "asg"} - c.save(k2, range(2)) - self.assertEqual(c.get(k1), range(5)) - self.assertEqual(c.get(k2), range(2)) - - c2 = cache.FileCacheManager(Namespace(cache_period=60, cache=t.name)) - self.assertTrue(c2.load()) - self.assertEqual(c2.get(k1), range(5)) - self.assertEqual(c2.get(k2), range(2)) - - def test_get(self): - # mock the pick and set it to the data variable - test_pickle = pickle.dumps( - {pickle.dumps(self.test_key): self.test_value}, protocol=2 - ) - self.test_cache.data = pickle.loads(test_pickle) - - # assert - self.assertEqual(self.test_cache.get(self.test_key), self.test_value) - self.assertEqual(self.test_cache.get(self.bad_key), None) - - def test_load(self): - t = self.temporary_file_with_cleanup(suffix=".cache") - - load_config = Namespace(cache_period=0, cache=t.name) - load_cache = cache.FileCacheManager(load_config) - self.assertFalse(load_cache.load()) - load_cache.data = {"key": "value"} - self.assertTrue(load_cache.load()) - - @mock.patch.object(cache.os, "makedirs") - @mock.patch.object(cache.os.path, "exists") - @mock.patch.object(cache.pickle, "dump") - @mock.patch.object(cache.pickle, "dumps") - def test_save_exists(self, mock_dumps, mock_dump, mock_exists, mock_mkdir): - # path exists then we dont need to create the folder - mock_exists.return_value = True - # tempfile to hold the pickle - temp_cache_file = self.temporary_file_with_cleanup() - - self.test_cache.cache_path = temp_cache_file.name - # make the call - self.test_cache.save(self.test_key, self.test_value) - - # assert if file already exists - self.assertFalse(mock_mkdir.called) - self.assertTrue(mock_dumps.called) - self.assertTrue(mock_dump.called) - - # mkdir should NOT be called, but pickles should - self.assertEqual(mock_mkdir.call_count, 0) - self.assertEqual(mock_dump.call_count, 1) - self.assertEqual(mock_dumps.call_count, 1) - - @mock.patch.object(cache.os, "makedirs") - @mock.patch.object(cache.os.path, "exists") - @mock.patch.object(cache.pickle, "dump") - @mock.patch.object(cache.pickle, "dumps") - def test_save_doesnt_exists(self, mock_dumps, mock_dump, mock_exists, mock_mkdir): - temp_cache_file = self.temporary_file_with_cleanup() - - self.test_cache.cache_path = temp_cache_file.name - - # path doesnt exists then we will create the folder - # raise some sort of exception in the try - mock_exists.return_value = False - mock_dump.side_effect = Exception("Error") - mock_mkdir.side_effect = Exception("Error") - - # make the call - self.test_cache.save(self.test_key, self.test_value) - - # assert if file doesnt exists - self.assertTrue(mock_mkdir.called) - self.assertTrue(mock_dumps.called) - self.assertTrue(mock_dump.called) - - # all 3 should be called once - self.assertEqual(mock_mkdir.call_count, 1) - self.assertEqual(mock_dump.call_count, 1) - self.assertEqual(mock_dumps.call_count, 1) - - def temporary_file_with_cleanup(self, **kwargs): - """ - NamedTemporaryFile with delete=True has - significantly different behavior on Windows - so we utilize delete=False to simplify maintaining - compatibility. - """ - t = tempfile.NamedTemporaryFile(delete=False, **kwargs) - - self.addCleanup(os.unlink, t.name) - self.addCleanup(t.close) - return t + mem_cache.close() + + +def test_sqlkv(tmp_path): + kv = cache.SqlKvCache(config.Bag(cache=tmp_path / "cache.db", cache_period=60)) + kv.load() + k1 = {"account": "12345678901234", "region": "us-west-2", "resource": "ec2"} + v1 = [{'id': 'a'}, {'id': 'b'}] + + assert kv.get(k1) is None + kv.save(k1, v1) + assert kv.get(k1) == v1 + kv.close() + + +def test_sqlkv_get_expired(tmp_path): + kv = cache.SqlKvCache(config.Bag(cache=tmp_path / "cache.db", cache_period=60)) + kv.load() + kv1 = {'a': 'b', 'c': 'd'} + kv.save(kv1, kv1, datetime.utcnow() - timedelta(days=10)) + assert kv.get(kv1) is None + + +def test_sqlkv_load_gc(tmp_path): + kv = cache.SqlKvCache(config.Bag(cache=tmp_path / "cache.db", cache_period=60)) + + # seed old values with manual connection + kv.conn = sqlite3.connect(kv.cache_path) + kv.conn.execute(kv.create_table) + kv1 = {'a': 'b', 'c': 'd'} + kv2 = {'b': 'a', 'd': 'c'} + kv.save(kv1, kv1, datetime.utcnow() - timedelta(days=10)) + kv.save(kv2, kv2, datetime.utcnow() - timedelta(minutes=5)) + + kv.load() + assert kv.get(kv1) is None + assert kv.get(kv2) == kv2 + + +def test_sqlkv_parent_dir_create(tmp_path): + cache_path = tmp_path / ".cache" / "cache.db" + kv = cache.SqlKvCache(config.Bag(cache=cache_path, cache_period=60)) + kv.load() + assert os.path.exists(os.path.dirname(cache_path)) + + +@pytest.mark.skipif( + sys.platform == 'win32', + reason="windows can't remove a recently created but closed file") +def test_sqlkv_convert(tmp_path): + cache_path = tmp_path / "cache2.db" + with open(cache_path, 'wb') as fh: + pickle.dump({'kv': 'abc'}, fh) + fh.close() + kv = cache.SqlKvCache(config.Bag(cache=cache_path, cache_period=60)) + kv.load() + kv.close() + with open(cache_path, 'rb') as fh: + assert fh.read(15) == b"SQLite format 3" diff --git a/tests/test_iam.py b/tests/test_iam.py index 7e830b775ef..412c729e25a 100644 --- a/tests/test_iam.py +++ b/tests/test_iam.py @@ -1,7 +1,6 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 import json -import datetime import os import mock import tempfile @@ -9,8 +8,8 @@ from unittest import TestCase from .common import load_data, BaseTest, functional -from .test_offhours import mock_datetime_now +import freezegun import pytest from pytest_terraform import terraform from dateutil import parser @@ -120,7 +119,7 @@ def test_credential_access_key_multifilter_delete(self): 'matched': True}]}, session_factory=factory) - with mock_datetime_now(parser.parse("2020-01-01"), datetime): + with freezegun.freeze_time('2020-01-01'): resources = p.run() self.assertEqual(len(resources), 1) self.assertEqual(len(resources[0]['c7n:matched-keys']), 1) @@ -240,7 +239,7 @@ def test_access_key_last_service(self): session_factory=session_factory, cache=True, ) - with mock_datetime_now(parser.parse("2016-11-25T20:27:00+00:00"), datetime): + with freezegun.freeze_time("2016-11-25T20:27"): resources = p.run() self.assertEqual(len(resources), 1) self.assertEqual(sorted([r["UserName"] for r in resources]), ["kapil"]) @@ -271,7 +270,7 @@ def test_old_console_users(self): cache=True, ) - with mock_datetime_now(parser.parse("2016-11-25T20:27:00+00:00"), datetime): + with freezegun.freeze_time("2016-11-25T20:27:00+00:00"): resources = p.run() self.assertEqual(len(resources), 3) self.assertEqual( diff --git a/tools/c7n_azure/c7n_azure/filters.py b/tools/c7n_azure/c7n_azure/filters.py index ea9484b2f74..a23a3a551f3 100644 --- a/tools/c7n_azure/c7n_azure/filters.py +++ b/tools/c7n_azure/c7n_azure/filters.py @@ -1012,6 +1012,5 @@ def __init__(self, data, manager=None): def process(self, resources, event=None): parent_resources = self.parent_filter.process(self.parent_manager.resources()) parent_resources_ids = [p['id'] for p in parent_resources] - parent_key = self.manager.resource_type.parent_key return [r for r in resources if r[parent_key] in parent_resources_ids] diff --git a/tools/c7n_azure/c7n_azure/query.py b/tools/c7n_azure/c7n_azure/query.py index b8186047a1b..091c4928109 100644 --- a/tools/c7n_azure/c7n_azure/query.py +++ b/tools/c7n_azure/c7n_azure/query.py @@ -159,7 +159,6 @@ def filter(self, resource_manager, **params): .format(parent[parents.resource_type.id], e)) if m.raise_on_exception: raise e - return results @@ -282,6 +281,8 @@ def resources(self, query=None, augment=True): resources = self.augment(resources) self._cache.save(cache_key, resources) + self._cache.close() + with self.ctx.tracer.subsegment('filter'): resource_count = len(resources) resources = self.filter_resources(resources) diff --git a/tools/c7n_gcp/c7n_gcp/query.py b/tools/c7n_gcp/c7n_gcp/query.py index 689d79db97c..e436962726d 100644 --- a/tools/c7n_gcp/c7n_gcp/query.py +++ b/tools/c7n_gcp/c7n_gcp/query.py @@ -188,14 +188,28 @@ def get_resource_query(self): def resources(self, query=None): q = query or self.get_resource_query() - key = self.get_cache_key(q) - resources = self._fetch_resources(q) - self._cache.save(key, resources) - + cache_key = self.get_cache_key(q) + resources = None + + if self._cache.load(): + resources = self._cache.get(cache_key) + if resources is not None: + self.log.debug("Using cached %s: %d" % ( + "%s.%s" % (self.__class__.__module__, + self.__class__.__name__), + len(resources))) + + if resources is None: + with self.ctx.tracer.subsegment('resource-fetch'): + resources = self._fetch_resources(q) + self._cache.save(cache_key, resources) + + self._cache.close() resource_count = len(resources) - resources = self.filter_resources(resources) + with self.ctx.tracer.subsegment('filter'): + resources = self.filter_resources(resources) - # Check if we're out of a policies execution limits. + # Check resource limits if we're the current policy execution. if self.data == self.ctx.policy.data: self.check_resource_limit(len(resources), resource_count) return resources