one groupby per build thread + new resolver class

lektor_groupby/groupby.py

@@ -1,11 +1,11 @@
 from lektor.builder import Builder, PathCache
-from lektor.db import Record
+from lektor.db import Record  # typing
-from lektor.sourceobj import SourceObject
+from lektor.sourceobj import SourceObject  # typing
-from lektor.utils import build_url
 
-from typing import Set, Dict, List, Optional, Tuple
+from typing import Set, List
-from .vobj import GroupBySource
+from .vobj import GroupBySource  # typing
 from .config import Config, AnyConfig
+from .resolver import Resolver  # typing
 from .watcher import Watcher
 
 
@@ -19,11 +19,6 @@ class GroupBy:
     def __init__(self) -> None:
         self._watcher = []  # type: List[Watcher]
         self._results = []  # type: List[GroupBySource]
-        self._resolver = {}  # type: Dict[str, Tuple[str, Config]]
-
-    # ----------------
-    #   Add observer
-    # ----------------
 
     def add_watcher(self, key: str, config: AnyConfig) -> Watcher:
         ''' Init Config and add to watch list. '''
@@ -31,24 +26,14 @@ class GroupBy:
         self._watcher.append(w)
         return w
 
-    # -----------
-    #   Builder
-    # -----------
-
-    def clear_previous_results(self) -> None:
-        ''' Reset prvious results. Must be called before each build. '''
-        self._watcher.clear()
-        self._results.clear()
-        self._resolver.clear()
-
     def get_dependencies(self) -> Set[str]:
         deps = set()  # type: Set[str]
         for w in self._watcher:
             deps.update(w.config.dependencies)
         return deps
 
-    def make_cluster(self, builder: Builder) -> None:
+    def queue_all(self, builder: Builder) -> None:
-        ''' Iterate over all children and perform groupby. '''
+        ''' Iterate full site-tree and queue all children. '''
         # remove disabled watchers
         self._watcher = [w for w in self._watcher if w.config.enabled]
         if not self._watcher:
@@ -65,14 +50,6 @@ class GroupBy:
                 queue.extend(record.attachments)  # type: ignore[attr-defined]
             if hasattr(record, 'children'):
                 queue.extend(record.children)  # type: ignore[attr-defined]
-        # build artifacts
-        for w in self._watcher:
-            root = builder.pad.get(w.config.root)
-            for vobj in w.iter_sources(root):
-                self._results.append(vobj)
-                if vobj.slug:
-                    self._resolver[vobj.url_path] = (vobj.group, w.config)
-        self._watcher.clear()
 
     def queue_now(self, node: SourceObject) -> None:
         ''' Process record immediatelly (No-Op if already processed). '''
@@ -81,6 +58,15 @@ class GroupBy:
                 if w.should_process(node):
                     w.process(node)
 
+    def make_cluster(self, builder: Builder, resolver: Resolver) -> None:
+        ''' Perform groupby, iter over sources with watcher callback. '''
+        for w in self._watcher:
+            root = builder.pad.get(w.config.root)
+            for vobj in w.iter_sources(root):
+                self._results.append(vobj)
+                resolver.add(vobj)
+        self._watcher.clear()
+
     def build_all(self, builder: Builder) -> None:
         ''' Create virtual objects and build sources. '''
         path_cache = PathCache(builder.env)
@@ -89,28 +75,3 @@ class GroupBy:
                 builder.build(vobj, path_cache)
         del path_cache
         self._results.clear()  # garbage collect weak refs
-
-    # -----------------
-    #   Path resolver
-    # -----------------
-
-    def resolve_dev_server_path(self, node: SourceObject, pieces: List[str]) \
-            -> Optional[GroupBySource]:
-        ''' Dev server only: Resolves path/ -> path/index.html '''
-        if isinstance(node, Record):
-            rv = self._resolver.get(build_url([node.url_path] + pieces))
-            if rv:
-                return GroupBySource(node, group=rv[0], config=rv[1])
-        return None
-
-    def resolve_virtual_path(self, node: SourceObject, pieces: List[str]) \
-            -> Optional[GroupBySource]:
-        ''' Admin UI only: Prevent server error and null-redirect. '''
-        if isinstance(node, Record) and len(pieces) >= 2:
-            path = node['_path']  # type: str
-            key, grp, *_ = pieces
-            for group, conf in self._resolver.values():
-                if key == conf.key and path == conf.root:
-                    if conf.slugify(group) == grp:
-                        return GroupBySource(node, group, conf)
-        return None

lektor_groupby/plugin.py

@@ -3,10 +3,11 @@ from lektor.db import Page  # typing
 from lektor.pluginsystem import Plugin  # subclass
 from lektor.sourceobj import SourceObject  # typing
 
-from typing import List, Optional, Iterator, Any
+from typing import Iterator, Any
 from .vobj import GroupBySource, GroupByBuildProgram, VPATH, VGroups
 from .groupby import GroupBy
 from .pruner import prune
+from .resolver import Resolver
 from .watcher import GroupByCallbackArgs  # typing
 
 
@@ -15,28 +16,18 @@ class GroupByPlugin(Plugin):
     description = 'Cluster arbitrary records with field attribute keyword.'
 
     def on_setup_env(self, **extra: Any) -> None:
-        self.creator = GroupBy()
+        self.resolver = Resolver(self.env)
         self.env.add_build_program(GroupBySource, GroupByBuildProgram)
         self.env.jinja_env.filters.update(vgroups=VGroups.iter)
 
-        # resolve /tag/rss/ -> /tag/rss/index.html (local server only)
+    def _load_quick_config(self, groupby: GroupBy) -> None:
-        @self.env.urlresolver
-        def a(node: SourceObject, parts: List[str]) -> Optional[GroupBySource]:
-            return self.creator.resolve_dev_server_path(node, parts)
-
-        # resolve virtual objects in admin UI
-        @self.env.virtualpathresolver(VPATH.lstrip('@'))
-        def b(node: SourceObject, parts: List[str]) -> Optional[GroupBySource]:
-            return self.creator.resolve_virtual_path(node, parts)
-
-    def _load_quick_config(self) -> None:
         ''' Load config file quick listeners. '''
         config = self.get_config()
         for key in config.sections():
             if '.' in key:  # e.g., key.fields and key.key_map
                 continue
 
-            watcher = self.creator.add_watcher(key, config)
+            watcher = groupby.add_watcher(key, config)
             split = config.get(key + '.split')  # type: str
 
             @watcher.grouping()
@@ -47,23 +38,32 @@ class GroupByPlugin(Plugin):
                 if isinstance(val, (list, map)):
                     yield from val
 
-    def on_before_build_all(self, builder: Builder, **extra: Any) -> None:
+    def _init_once(self, builder: Builder) -> GroupBy:
-        self.creator.clear_previous_results()
+        try:
-        self._load_quick_config()
+            return builder.__groupby  # type:ignore[attr-defined,no-any-return]
-        # let other plugins register their @groupby.watch functions
+        except AttributeError:
-        self.emit('before-build-all', groupby=self.creator, builder=builder)
+            groupby = GroupBy()
-        self.config_dependencies = self.creator.get_dependencies()
+            builder.__groupby = groupby  # type: ignore[attr-defined]
-        self.creator.make_cluster(builder)
 
-    def on_before_build(self, source: SourceObject, **extra: Any) -> None:
+        self.resolver.reset()
+        self._load_quick_config(groupby)
+        # let other plugins register their @groupby.watch functions
+        self.emit('before-build-all', groupby=groupby, builder=builder)
+        self.config_dependencies = groupby.get_dependencies()
+        groupby.queue_all(builder)
+        groupby.make_cluster(builder, self.resolver)
+        return groupby
+
+    def on_before_build(self, builder: Builder, source: SourceObject,
+                        **extra: Any) -> None:
         # before-build may be called before before-build-all (issue #1017)
         # make sure it is evaluated immediatelly
         if isinstance(source, Page):
-            self.creator.queue_now(source)
+            self._init_once(builder)
 
-    def on_after_build_all(self, builder: Builder, **extra: Any) -> None:
+    def on_after_build_all(self, builder: Builder, **extra: object) -> None:
-        self.creator.build_all(builder)
+        self._init_once(builder).build_all(builder)
 
-    def on_after_prune(self, builder: Builder, **extra: Any) -> None:
+    def on_after_prune(self, builder: Builder, **extra: object) -> None:
         # TODO: find a better way to prune unreferenced elements
         prune(builder, VPATH)

lektor_groupby/resolver.py

@@ -0,0 +1,50 @@
+from lektor.db import Record
+from lektor.environment import Environment
+from lektor.sourceobj import SourceObject
+from lektor.utils import build_url
+
+from typing import Dict, List, Tuple, Optional
+from .config import Config  # typing
+from .vobj import GroupBySource, VPATH
+
+
+class Resolver:
+    '''
+    Resolve virtual paths and urls ending in /.
+    Init will subscribe to @urlresolver and @virtualpathresolver.
+    '''
+
+    def __init__(self, env: Environment) -> None:
+        self._data = {}  # type: Dict[str, Tuple[str, Config]]
+
+        # Local server only: resolve /tag/rss/ -> /tag/rss/index.html
+        @env.urlresolver
+        def dev_server_path(node: SourceObject, pieces: List[str]) \
+                -> Optional[GroupBySource]:
+            if isinstance(node, Record):
+                rv = self._data.get(build_url([node.url_path] + pieces))
+                if rv:
+                    return GroupBySource(node, group=rv[0], config=rv[1])
+            return None
+
+        # Admin UI only: Prevent server error and null-redirect.
+        @env.virtualpathresolver(VPATH.lstrip('@'))
+        def virtual_path(node: SourceObject, pieces: List[str]) \
+                -> Optional[GroupBySource]:
+            if isinstance(node, Record) and len(pieces) >= 2:
+                path = node['_path']  # type: str
+                key, grp, *_ = pieces
+                for group, conf in self._data.values():
+                    if key == conf.key and path == conf.root:
+                        if conf.slugify(group) == grp:
+                            return GroupBySource(node, group, conf)
+            return None
+
+    def reset(self) -> None:
+        ''' Clear previously recorded virtual objects. '''
+        self._data.clear()
+
+    def add(self, vobj: GroupBySource) -> None:
+        ''' Track new virtual object (only if slug is set). '''
+        if vobj.slug:
+            self._data[vobj.url_path] = (vobj.group, vobj.config)

lektor_groupby/vobj.py

@@ -158,7 +158,7 @@ class VGroups:
         record: Record,
         *keys: str,
         recursive: bool = False
-    ) -> Iterator['GroupBySource']:
+    ) -> Iterator[GroupBySource]:
         ''' Extract all referencing groupby virtual objects from a page. '''
         ctx = get_ctx()
         # manage dependencies