2nd level caching for zcatalog by means of plone.memoize

setup.py

@@ -69,6 +69,7 @@
         'zope.interface',
         'zope.schema',
         'zope.testing',
+        'plone.memoize',
     ],
     include_package_data=True,
     zip_safe=False,

src/Products/PluginIndexes/DateRangeIndex/DateRangeIndex.py

@@ -250,6 +250,7 @@ def getRequestCacheKey(self, record, resultset=None):
         # unique index identifier
         iid = '_{0}_{1}_{2}'.format(self.__class__.__name__,
                                     self.id, self.getCounter())
+
         # record identifier
         if resultset is None:
             rid = '_{0}'.format(tid)

src/Products/PluginIndexes/PathIndex/PathIndex.py

@@ -31,14 +31,16 @@
     IQueryIndex,
     ISortIndex,
     IUniqueValueIndex,
+    IIndexCounter,
 )
 from Products.PluginIndexes.util import safe_callable
 from Products.ZCatalog.query import IndexQuery
 
 LOG = getLogger('Zope.PathIndex')
 
 
-@implementer(IPathIndex, IQueryIndex, IUniqueValueIndex, ISortIndex)
+@implementer(IPathIndex, IQueryIndex, IUniqueValueIndex,
+             ISortIndex, IIndexCounter)
 class PathIndex(Persistent, SimpleItem):
 
     """Index for paths returned by getPhysicalPath.
@@ -61,6 +63,7 @@ class PathIndex(Persistent, SimpleItem):
     operators = ('or', 'and')
     useOperator = 'or'
     query_options = ('query', 'level', 'operator')
+    _counter = None
 
     manage_options = (
         {'label': 'Settings', 'action': 'manage_main'},
@@ -129,6 +132,7 @@ def index_object(self, docid, obj, threshold=100):
         for i in range(len(comps)):
             self.insertEntry(comps[i], docid, i)
         self._unindex[docid] = path
+        self._increment_counter()
         return 1
 
     def unindex_object(self, docid):
@@ -156,6 +160,7 @@ def unindex_object(self, docid):
                           'with id %s failed', docid)
 
         self._length.change(-1)
+        self._increment_counter()
         del self._unindex[docid]
 
     def _apply_index(self, request):
@@ -188,6 +193,15 @@ def query_index(self, record, resultset=None):
             return res
         return IISet()
 
+    def _increment_counter(self):
+        if self._counter is None:
+            self._counter = Length()
+        self._counter.change(1)
+
+    def getCounter(self):
+        """Return a counter which is increased on index changes"""
+        return self._counter is not None and self._counter() or 0
+
     def numObjects(self):
         """ See IPluggableIndex.
         """
@@ -205,6 +219,10 @@ def clear(self):
         self._index = OOBTree()
         self._unindex = IOBTree()
         self._length = Length(0)
+        if self._counter is None:
+            self._counter = Length(0)
+        else:
+            self._increment_counter()
 
     # IUniqueValueIndex implementation
 

src/Products/PluginIndexes/PathIndex/tests.py

@@ -533,3 +533,23 @@ def test__search_w_level_0(self):
         self.assertEqual(list(index._search('bb', 1)), [1])
         self.assertEqual(list(index._search('aa/bb', 0)), [1])
         self.assertEqual(list(index._search('aa/bb', 1)), [])
+
+    def test_getCounter(self):
+        index = self._makeOne()
+
+        self.assertEqual(index.getCounter(), 0)
+
+        doc = Dummy('/aa/bb')
+        index.index_object(1, doc)
+        self.assertEqual(index.getCounter(), 1)
+
+        index.unindex_object(1)
+        self.assertEqual(index.getCounter(), 2)
+
+        # unknown id
+        index.unindex_object(1)
+        self.assertEqual(index.getCounter(), 2)
+
+        # clear changes the index
+        index.clear()
+        self.assertEqual(index.getCounter(), 3)

src/Products/PluginIndexes/TopicIndex/FilteredSet.py

@@ -41,10 +41,7 @@ def index_object(self, documentId, obj):
         raise RuntimeError('index_object not defined')
 
     def unindex_object(self, documentId):
-        try:
-            self.ids.remove(documentId)
-        except KeyError:
-            pass
+        self.ids.remove(documentId)
 
     def getId(self):
         return self.id
@@ -78,19 +75,23 @@ class PythonFilteredSet(FilteredSetBase):
     meta_type = 'PythonFilteredSet'
 
     def index_object(self, documentId, o):
+        res = 0
         try:
             if RestrictionCapableEval(self.expr).eval({'o': o}):
                 self.ids.insert(documentId)
+                res = 1
             else:
                 try:
                     self.ids.remove(documentId)
+                    res = 1
                 except KeyError:
                     pass
         except ConflictError:
             raise
         except Exception:
             LOG.warn('eval() failed Object: %s, expr: %s',
                      o.getId(), self.expr, exc_info=sys.exc_info())
+        return res
 
 
 def factory(f_id, f_type, expr):

src/Products/PluginIndexes/TopicIndex/TopicIndex.py

@@ -18,13 +18,15 @@
 from BTrees.IIBTree import intersection
 from BTrees.IIBTree import union
 from BTrees.OOBTree import OOBTree
+from BTrees.Length import Length
 from OFS.SimpleItem import SimpleItem
 from Persistence import Persistent
 from zope.interface import implementer
 
 from Products.PluginIndexes.interfaces import (
     IQueryIndex,
     ITopicIndex,
+    IIndexCounter,
 )
 from Products.PluginIndexes.TopicIndex.FilteredSet import factory
 from Products.ZCatalog.query import IndexQuery
@@ -33,7 +35,7 @@
 LOG = getLogger('Zope.TopicIndex')
 
 
-@implementer(ITopicIndex, IQueryIndex)
+@implementer(ITopicIndex, IQueryIndex, IIndexCounter)
 class TopicIndex(Persistent, SimpleItem):
     """A TopicIndex maintains a set of FilteredSet objects.
 
@@ -44,6 +46,7 @@ class TopicIndex(Persistent, SimpleItem):
     meta_type = 'TopicIndex'
     zmi_icon = 'fas fa-info-circle'
     query_options = ('query', 'operator')
+    _counter = None
 
     manage_options = (
         {'label': 'FilteredSets', 'action': 'manage_main'},
@@ -52,6 +55,7 @@ class TopicIndex(Persistent, SimpleItem):
     def __init__(self, id, caller=None):
         self.id = id
         self.filteredSets = OOBTree()
+        self._counter = Length()
         self.operators = ('or', 'and')
         self.defaultOperator = 'or'
 
@@ -61,22 +65,45 @@ def getId(self):
     def clear(self):
         for fs in self.filteredSets.values():
             fs.clear()
+        self._increment_counter()
 
     def index_object(self, docid, obj, threshold=100):
         """ hook for (Z)Catalog """
+        res = 0
         for fid, filteredSet in self.filteredSets.items():
-            filteredSet.index_object(docid, obj)
-        return 1
+            res += filteredSet.index_object(docid, obj)
+
+        if res > 0:
+            self._increment_counter()
+            return 1
+
+        return 0
 
     def unindex_object(self, docid):
         """ hook for (Z)Catalog """
+        res = 0
         for fs in self.filteredSets.values():
             try:
                 fs.unindex_object(docid)
+                res += 1
             except KeyError:
                 LOG.debug('Attempt to unindex document'
                           ' with id %s failed', docid)
-        return 1
+
+        if res > 0:
+            self._increment_counter()
+            return 1
+
+        return 0
+
+    def _increment_counter(self):
+        if self._counter is None:
+            self._counter = Length()
+        self._counter.change(1)
+
+    def getCounter(self):
+        """Return a counter which is increased on index changes"""
+        return self._counter is not None and self._counter() or 0
 
     def numObjects(self):
         """Return the number of indexed objects."""

src/Products/PluginIndexes/TopicIndex/tests.py

@@ -92,3 +92,23 @@ def testRemoval(self):
         self.TI.index_object(1, Obj('1', 'doc2'))
         self._searchOr('doc1', [2])
         self._searchOr('doc2', [1, 3, 4])
+
+    def test_getCounter(self):
+        index = self.TI
+        index._counter.set(0)
+
+        self.assertEqual(index.getCounter(), 0)
+
+        index.index_object(1, Obj(1, 'doc1'))
+        self.assertEqual(index.getCounter(), 1)
+
+        index.unindex_object(1)
+        self.assertEqual(index.getCounter(), 2)
+
+        # unknown id
+        index.unindex_object(1)
+        self.assertEqual(index.getCounter(), 2)
+
+        # clear changes the index
+        index.clear()
+        self.assertEqual(index.getCounter(), 3)

src/Products/PluginIndexes/interfaces.py

@@ -283,3 +283,10 @@ def make_query(query):
 
     def getIndexNames():
         """ returns index names that are optimized by index """
+
+
+class IIndexCounter(Interface):
+    """ Invalidation helper API for pluggable indexes"""
+
+    def getCounter():
+        """Return a counter which is increased on index changes"""

src/Products/PluginIndexes/unindex.py

@@ -40,6 +40,7 @@
     ISortIndex,
     IUniqueValueIndex,
     IRequestCacheIndex,
+    IIndexCounter,
 )
 from Products.PluginIndexes.util import safe_callable
 from Products.ZCatalog.query import IndexQuery
@@ -49,7 +50,7 @@
 
 
 @implementer(ILimitedResultIndex, IQueryIndex, IUniqueValueIndex,
-             ISortIndex, IRequestCacheIndex)
+             ISortIndex, IRequestCacheIndex, IIndexCounter)
 class UnIndex(SimpleItem):
     """Simple forward and reverse index.
     """
@@ -404,6 +405,7 @@ def getRequestCacheKey(self, record, resultset=None):
         # unique index identifier
         iid = '_{0}_{1}_{2}'.format(self.__class__.__name__,
                                     self.id, self.getCounter())
+
         return (iid, rid)
 
     def _apply_index(self, request, resultset=None):

src/Products/ZCatalog/Catalog.py

@@ -38,6 +38,7 @@
 from Products.PluginIndexes.util import safe_callable
 from Products.ZCatalog.CatalogBrains import AbstractCatalogBrain, NoBrainer
 from Products.ZCatalog.plan import CatalogPlan
+from Products.ZCatalog.cache import cache
 from Products.ZCatalog.ProgressHandler import ZLogHandler
 from Products.ZCatalog.query import IndexQuery
 
@@ -594,6 +595,28 @@ def _search_index(self, cr, index_id, query, rs):
 
         return rs
 
+    @cache
+    def _apply_query_plan(self, cr, query):
+
+        plan = cr.plan()
+        if not plan:
+            plan = self._sorted_search_indexes(query)
+
+        rs = None  # result set
+        for index_id in plan:
+            # The actual core loop over all indices.
+            if index_id not in self.indexes:
+                # We can have bogus keys or the plan can contain
+                # index names that have been removed in the
+                # meantime.
+                continue
+
+            rs = self._search_index(cr, index_id, query, rs)
+            if not rs:
+                break
+
+        return rs
+
     def search(self, query,
                sort_index=None, reverse=False, limit=None, merge=True):
         """Iterate through the indexes, applying the query to each one. If
@@ -619,21 +642,7 @@ def search(self, query,
         cr = self.getCatalogPlan(query)
         cr.start()
 
-        plan = cr.plan()
-        if not plan:
-            plan = self._sorted_search_indexes(query)
-
-        rs = None  # result set
-        for index_id in plan:
-            # The actual core loop over all indices.
-            if index_id not in self.indexes:
-                # We can have bogus keys or the plan can contain index names
-                # that have been removed in the meantime.
-                continue
-
-            rs = self._search_index(cr, index_id, query, rs)
-            if not rs:
-                break
+        rs = self._apply_query_plan(cr, query)
 
         if not rs:
             # None of the indexes found anything to do with the query.