From 340bc6611bd597087f01f015533f35f0b9d1d0cc Mon Sep 17 00:00:00 2001 From: relikd Date: Mon, 11 Apr 2022 01:41:17 +0200 Subject: [PATCH] one groupby per build thread + new resolver class --- lektor_groupby/groupby.py | 71 +++++++++----------------------------- lektor_groupby/plugin.py | 52 ++++++++++++++-------------- lektor_groupby/resolver.py | 50 +++++++++++++++++++++++++++ lektor_groupby/vobj.py | 2 +- 4 files changed, 93 insertions(+), 82 deletions(-) create mode 100644 lektor_groupby/resolver.py diff --git a/lektor_groupby/groupby.py b/lektor_groupby/groupby.py index fc9d9e2..abf918a 100644 --- a/lektor_groupby/groupby.py +++ b/lektor_groupby/groupby.py @@ -1,11 +1,11 @@ from lektor.builder import Builder, PathCache -from lektor.db import Record -from lektor.sourceobj import SourceObject -from lektor.utils import build_url +from lektor.db import Record # typing +from lektor.sourceobj import SourceObject # typing -from typing import Set, Dict, List, Optional, Tuple -from .vobj import GroupBySource +from typing import Set, List +from .vobj import GroupBySource # typing from .config import Config, AnyConfig +from .resolver import Resolver # typing from .watcher import Watcher @@ -19,11 +19,6 @@ class GroupBy: def __init__(self) -> None: self._watcher = [] # type: List[Watcher] self._results = [] # type: List[GroupBySource] - self._resolver = {} # type: Dict[str, Tuple[str, Config]] - - # ---------------- - # Add observer - # ---------------- def add_watcher(self, key: str, config: AnyConfig) -> Watcher: ''' Init Config and add to watch list. ''' @@ -31,24 +26,14 @@ class GroupBy: self._watcher.append(w) return w - # ----------- - # Builder - # ----------- - - def clear_previous_results(self) -> None: - ''' Reset prvious results. Must be called before each build. ''' - self._watcher.clear() - self._results.clear() - self._resolver.clear() - def get_dependencies(self) -> Set[str]: deps = set() # type: Set[str] for w in self._watcher: deps.update(w.config.dependencies) return deps - def make_cluster(self, builder: Builder) -> None: - ''' Iterate over all children and perform groupby. ''' + def queue_all(self, builder: Builder) -> None: + ''' Iterate full site-tree and queue all children. ''' # remove disabled watchers self._watcher = [w for w in self._watcher if w.config.enabled] if not self._watcher: @@ -65,14 +50,6 @@ class GroupBy: queue.extend(record.attachments) # type: ignore[attr-defined] if hasattr(record, 'children'): queue.extend(record.children) # type: ignore[attr-defined] - # build artifacts - for w in self._watcher: - root = builder.pad.get(w.config.root) - for vobj in w.iter_sources(root): - self._results.append(vobj) - if vobj.slug: - self._resolver[vobj.url_path] = (vobj.group, w.config) - self._watcher.clear() def queue_now(self, node: SourceObject) -> None: ''' Process record immediatelly (No-Op if already processed). ''' @@ -81,6 +58,15 @@ class GroupBy: if w.should_process(node): w.process(node) + def make_cluster(self, builder: Builder, resolver: Resolver) -> None: + ''' Perform groupby, iter over sources with watcher callback. ''' + for w in self._watcher: + root = builder.pad.get(w.config.root) + for vobj in w.iter_sources(root): + self._results.append(vobj) + resolver.add(vobj) + self._watcher.clear() + def build_all(self, builder: Builder) -> None: ''' Create virtual objects and build sources. ''' path_cache = PathCache(builder.env) @@ -89,28 +75,3 @@ class GroupBy: builder.build(vobj, path_cache) del path_cache self._results.clear() # garbage collect weak refs - - # ----------------- - # Path resolver - # ----------------- - - def resolve_dev_server_path(self, node: SourceObject, pieces: List[str]) \ - -> Optional[GroupBySource]: - ''' Dev server only: Resolves path/ -> path/index.html ''' - if isinstance(node, Record): - rv = self._resolver.get(build_url([node.url_path] + pieces)) - if rv: - return GroupBySource(node, group=rv[0], config=rv[1]) - return None - - def resolve_virtual_path(self, node: SourceObject, pieces: List[str]) \ - -> Optional[GroupBySource]: - ''' Admin UI only: Prevent server error and null-redirect. ''' - if isinstance(node, Record) and len(pieces) >= 2: - path = node['_path'] # type: str - key, grp, *_ = pieces - for group, conf in self._resolver.values(): - if key == conf.key and path == conf.root: - if conf.slugify(group) == grp: - return GroupBySource(node, group, conf) - return None diff --git a/lektor_groupby/plugin.py b/lektor_groupby/plugin.py index 9a7a11d..da6213d 100644 --- a/lektor_groupby/plugin.py +++ b/lektor_groupby/plugin.py @@ -3,10 +3,11 @@ from lektor.db import Page # typing from lektor.pluginsystem import Plugin # subclass from lektor.sourceobj import SourceObject # typing -from typing import List, Optional, Iterator, Any +from typing import Iterator, Any from .vobj import GroupBySource, GroupByBuildProgram, VPATH, VGroups from .groupby import GroupBy from .pruner import prune +from .resolver import Resolver from .watcher import GroupByCallbackArgs # typing @@ -15,28 +16,18 @@ class GroupByPlugin(Plugin): description = 'Cluster arbitrary records with field attribute keyword.' def on_setup_env(self, **extra: Any) -> None: - self.creator = GroupBy() + self.resolver = Resolver(self.env) self.env.add_build_program(GroupBySource, GroupByBuildProgram) self.env.jinja_env.filters.update(vgroups=VGroups.iter) - # resolve /tag/rss/ -> /tag/rss/index.html (local server only) - @self.env.urlresolver - def a(node: SourceObject, parts: List[str]) -> Optional[GroupBySource]: - return self.creator.resolve_dev_server_path(node, parts) - - # resolve virtual objects in admin UI - @self.env.virtualpathresolver(VPATH.lstrip('@')) - def b(node: SourceObject, parts: List[str]) -> Optional[GroupBySource]: - return self.creator.resolve_virtual_path(node, parts) - - def _load_quick_config(self) -> None: + def _load_quick_config(self, groupby: GroupBy) -> None: ''' Load config file quick listeners. ''' config = self.get_config() for key in config.sections(): if '.' in key: # e.g., key.fields and key.key_map continue - watcher = self.creator.add_watcher(key, config) + watcher = groupby.add_watcher(key, config) split = config.get(key + '.split') # type: str @watcher.grouping() @@ -47,23 +38,32 @@ class GroupByPlugin(Plugin): if isinstance(val, (list, map)): yield from val - def on_before_build_all(self, builder: Builder, **extra: Any) -> None: - self.creator.clear_previous_results() - self._load_quick_config() - # let other plugins register their @groupby.watch functions - self.emit('before-build-all', groupby=self.creator, builder=builder) - self.config_dependencies = self.creator.get_dependencies() - self.creator.make_cluster(builder) + def _init_once(self, builder: Builder) -> GroupBy: + try: + return builder.__groupby # type:ignore[attr-defined,no-any-return] + except AttributeError: + groupby = GroupBy() + builder.__groupby = groupby # type: ignore[attr-defined] - def on_before_build(self, source: SourceObject, **extra: Any) -> None: + self.resolver.reset() + self._load_quick_config(groupby) + # let other plugins register their @groupby.watch functions + self.emit('before-build-all', groupby=groupby, builder=builder) + self.config_dependencies = groupby.get_dependencies() + groupby.queue_all(builder) + groupby.make_cluster(builder, self.resolver) + return groupby + + def on_before_build(self, builder: Builder, source: SourceObject, + **extra: Any) -> None: # before-build may be called before before-build-all (issue #1017) # make sure it is evaluated immediatelly if isinstance(source, Page): - self.creator.queue_now(source) + self._init_once(builder) - def on_after_build_all(self, builder: Builder, **extra: Any) -> None: - self.creator.build_all(builder) + def on_after_build_all(self, builder: Builder, **extra: object) -> None: + self._init_once(builder).build_all(builder) - def on_after_prune(self, builder: Builder, **extra: Any) -> None: + def on_after_prune(self, builder: Builder, **extra: object) -> None: # TODO: find a better way to prune unreferenced elements prune(builder, VPATH) diff --git a/lektor_groupby/resolver.py b/lektor_groupby/resolver.py new file mode 100644 index 0000000..1bab0ac --- /dev/null +++ b/lektor_groupby/resolver.py @@ -0,0 +1,50 @@ +from lektor.db import Record +from lektor.environment import Environment +from lektor.sourceobj import SourceObject +from lektor.utils import build_url + +from typing import Dict, List, Tuple, Optional +from .config import Config # typing +from .vobj import GroupBySource, VPATH + + +class Resolver: + ''' + Resolve virtual paths and urls ending in /. + Init will subscribe to @urlresolver and @virtualpathresolver. + ''' + + def __init__(self, env: Environment) -> None: + self._data = {} # type: Dict[str, Tuple[str, Config]] + + # Local server only: resolve /tag/rss/ -> /tag/rss/index.html + @env.urlresolver + def dev_server_path(node: SourceObject, pieces: List[str]) \ + -> Optional[GroupBySource]: + if isinstance(node, Record): + rv = self._data.get(build_url([node.url_path] + pieces)) + if rv: + return GroupBySource(node, group=rv[0], config=rv[1]) + return None + + # Admin UI only: Prevent server error and null-redirect. + @env.virtualpathresolver(VPATH.lstrip('@')) + def virtual_path(node: SourceObject, pieces: List[str]) \ + -> Optional[GroupBySource]: + if isinstance(node, Record) and len(pieces) >= 2: + path = node['_path'] # type: str + key, grp, *_ = pieces + for group, conf in self._data.values(): + if key == conf.key and path == conf.root: + if conf.slugify(group) == grp: + return GroupBySource(node, group, conf) + return None + + def reset(self) -> None: + ''' Clear previously recorded virtual objects. ''' + self._data.clear() + + def add(self, vobj: GroupBySource) -> None: + ''' Track new virtual object (only if slug is set). ''' + if vobj.slug: + self._data[vobj.url_path] = (vobj.group, vobj.config) diff --git a/lektor_groupby/vobj.py b/lektor_groupby/vobj.py index 02d97ff..017ad09 100644 --- a/lektor_groupby/vobj.py +++ b/lektor_groupby/vobj.py @@ -158,7 +158,7 @@ class VGroups: record: Record, *keys: str, recursive: bool = False - ) -> Iterator['GroupBySource']: + ) -> Iterator[GroupBySource]: ''' Extract all referencing groupby virtual objects from a page. ''' ctx = get_ctx() # manage dependencies