relikd/lektor-groupby-plugin
1
0
Fork 0
Code Issues Pull Requests Releases Activity

one groupby per build thread + new resolver class

Browse Source
This commit is contained in:
relikd
2022-04-11 01:41:17 +02:00
parent 9dcd704283
commit 340bc6611b
4 changed files with 93 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
lektor_groupby/groupby.py
View File

@@ -1,11 +1,11 @@
from lektor.builder import Builder, PathCache
from lektor.db import Record
from lektor.sourceobj import SourceObject
from lektor.utils import build_url
from lektor.db import Record # typing
from lektor.sourceobj import SourceObject # typing
from typing import Set, Dict, List, Optional, Tuple
from .vobj import GroupBySource
from typing import Set, List
from .vobj import GroupBySource # typing
from .config import Config, AnyConfig
from .resolver import Resolver # typing
from .watcher import Watcher
@@ -19,11 +19,6 @@ class GroupBy:
def __init__(self) -> None:
self._watcher = [] # type: List[Watcher]
self._results = [] # type: List[GroupBySource]
self._resolver = {} # type: Dict[str, Tuple[str, Config]]
# ----------------
# Add observer
# ----------------
def add_watcher(self, key: str, config: AnyConfig) -> Watcher:
''' Init Config and add to watch list. '''
@@ -31,24 +26,14 @@ class GroupBy:
self._watcher.append(w)
return w
# -----------
# Builder
# -----------
def clear_previous_results(self) -> None:
''' Reset prvious results. Must be called before each build. '''
self._watcher.clear()
self._results.clear()
self._resolver.clear()
def get_dependencies(self) -> Set[str]:
deps = set() # type: Set[str]
for w in self._watcher:
deps.update(w.config.dependencies)
return deps
def make_cluster(self, builder: Builder) -> None:
''' Iterate over all children and perform groupby. '''
def queue_all(self, builder: Builder) -> None:
''' Iterate full site-tree and queue all children. '''
# remove disabled watchers
self._watcher = [w for w in self._watcher if w.config.enabled]
if not self._watcher:
@@ -65,14 +50,6 @@ class GroupBy:
queue.extend(record.attachments) # type: ignore[attr-defined]
if hasattr(record, 'children'):
queue.extend(record.children) # type: ignore[attr-defined]
# build artifacts
for w in self._watcher:
root = builder.pad.get(w.config.root)
for vobj in w.iter_sources(root):
self._results.append(vobj)
if vobj.slug:
self._resolver[vobj.url_path] = (vobj.group, w.config)
self._watcher.clear()
def queue_now(self, node: SourceObject) -> None:
''' Process record immediatelly (No-Op if already processed). '''
@@ -81,6 +58,15 @@ class GroupBy:
if w.should_process(node):
w.process(node)
def make_cluster(self, builder: Builder, resolver: Resolver) -> None:
''' Perform groupby, iter over sources with watcher callback. '''
for w in self._watcher:
root = builder.pad.get(w.config.root)
for vobj in w.iter_sources(root):
self._results.append(vobj)
resolver.add(vobj)
self._watcher.clear()
def build_all(self, builder: Builder) -> None:
''' Create virtual objects and build sources. '''
path_cache = PathCache(builder.env)
@@ -89,28 +75,3 @@ class GroupBy:
builder.build(vobj, path_cache)
del path_cache
self._results.clear() # garbage collect weak refs
# -----------------
# Path resolver
# -----------------
def resolve_dev_server_path(self, node: SourceObject, pieces: List[str]) \
-> Optional[GroupBySource]:
''' Dev server only: Resolves path/ -> path/index.html '''
if isinstance(node, Record):
rv = self._resolver.get(build_url([node.url_path] + pieces))
if rv:
return GroupBySource(node, group=rv[0], config=rv[1])
return None
def resolve_virtual_path(self, node: SourceObject, pieces: List[str]) \
-> Optional[GroupBySource]:
''' Admin UI only: Prevent server error and null-redirect. '''
if isinstance(node, Record) and len(pieces) >= 2:
path = node['_path'] # type: str
key, grp, *_ = pieces
for group, conf in self._resolver.values():
if key == conf.key and path == conf.root:
if conf.slugify(group) == grp:
return GroupBySource(node, group, conf)
return None

52
lektor_groupby/plugin.py
View File

@@ -3,10 +3,11 @@ from lektor.db import Page # typing
from lektor.pluginsystem import Plugin # subclass
from lektor.sourceobj import SourceObject # typing
from typing import List, Optional, Iterator, Any
from typing import Iterator, Any
from .vobj import GroupBySource, GroupByBuildProgram, VPATH, VGroups
from .groupby import GroupBy
from .pruner import prune
from .resolver import Resolver
from .watcher import GroupByCallbackArgs # typing
@@ -15,28 +16,18 @@ class GroupByPlugin(Plugin):
description = 'Cluster arbitrary records with field attribute keyword.'
def on_setup_env(self, **extra: Any) -> None:
self.creator = GroupBy()
self.resolver = Resolver(self.env)
self.env.add_build_program(GroupBySource, GroupByBuildProgram)
self.env.jinja_env.filters.update(vgroups=VGroups.iter)
# resolve /tag/rss/ -> /tag/rss/index.html (local server only)
@self.env.urlresolver
def a(node: SourceObject, parts: List[str]) -> Optional[GroupBySource]:
return self.creator.resolve_dev_server_path(node, parts)
# resolve virtual objects in admin UI
@self.env.virtualpathresolver(VPATH.lstrip('@'))
def b(node: SourceObject, parts: List[str]) -> Optional[GroupBySource]:
return self.creator.resolve_virtual_path(node, parts)
def _load_quick_config(self) -> None:
def _load_quick_config(self, groupby: GroupBy) -> None:
''' Load config file quick listeners. '''
config = self.get_config()
for key in config.sections():
if '.' in key: # e.g., key.fields and key.key_map
continue
watcher = self.creator.add_watcher(key, config)
watcher = groupby.add_watcher(key, config)
split = config.get(key + '.split') # type: str
@watcher.grouping()
@@ -47,23 +38,32 @@ class GroupByPlugin(Plugin):
if isinstance(val, (list, map)):
yield from val
def on_before_build_all(self, builder: Builder, **extra: Any) -> None:
self.creator.clear_previous_results()
self._load_quick_config()
# let other plugins register their @groupby.watch functions
self.emit('before-build-all', groupby=self.creator, builder=builder)
self.config_dependencies = self.creator.get_dependencies()
self.creator.make_cluster(builder)
def _init_once(self, builder: Builder) -> GroupBy:
try:
return builder.__groupby # type:ignore[attr-defined,no-any-return]
except AttributeError:
groupby = GroupBy()
builder.__groupby = groupby # type: ignore[attr-defined]
def on_before_build(self, source: SourceObject, **extra: Any) -> None:
self.resolver.reset()
self._load_quick_config(groupby)
# let other plugins register their @groupby.watch functions
self.emit('before-build-all', groupby=groupby, builder=builder)
self.config_dependencies = groupby.get_dependencies()
groupby.queue_all(builder)
groupby.make_cluster(builder, self.resolver)
return groupby
def on_before_build(self, builder: Builder, source: SourceObject,
**extra: Any) -> None:
# before-build may be called before before-build-all (issue #1017)
# make sure it is evaluated immediatelly
if isinstance(source, Page):
self.creator.queue_now(source)
self._init_once(builder)
def on_after_build_all(self, builder: Builder, **extra: Any) -> None:
self.creator.build_all(builder)
def on_after_build_all(self, builder: Builder, **extra: object) -> None:
self._init_once(builder).build_all(builder)
def on_after_prune(self, builder: Builder, **extra: Any) -> None:
def on_after_prune(self, builder: Builder, **extra: object) -> None:
# TODO: find a better way to prune unreferenced elements
prune(builder, VPATH)

50
lektor_groupby/resolver.py Normal file
View File

@@ -0,0 +1,50 @@
from lektor.db import Record
from lektor.environment import Environment
from lektor.sourceobj import SourceObject
from lektor.utils import build_url
from typing import Dict, List, Tuple, Optional
from .config import Config # typing
from .vobj import GroupBySource, VPATH
class Resolver:
'''
Resolve virtual paths and urls ending in /.
Init will subscribe to @urlresolver and @virtualpathresolver.
'''
def __init__(self, env: Environment) -> None:
self._data = {} # type: Dict[str, Tuple[str, Config]]
# Local server only: resolve /tag/rss/ -> /tag/rss/index.html
@env.urlresolver
def dev_server_path(node: SourceObject, pieces: List[str]) \
-> Optional[GroupBySource]:
if isinstance(node, Record):
rv = self._data.get(build_url([node.url_path] + pieces))
if rv:
return GroupBySource(node, group=rv[0], config=rv[1])
return None
# Admin UI only: Prevent server error and null-redirect.
@env.virtualpathresolver(VPATH.lstrip('@'))
def virtual_path(node: SourceObject, pieces: List[str]) \
-> Optional[GroupBySource]:
if isinstance(node, Record) and len(pieces) >= 2:
path = node['_path'] # type: str
key, grp, *_ = pieces
for group, conf in self._data.values():
if key == conf.key and path == conf.root:
if conf.slugify(group) == grp:
return GroupBySource(node, group, conf)
return None
def reset(self) -> None:
''' Clear previously recorded virtual objects. '''
self._data.clear()
def add(self, vobj: GroupBySource) -> None:
''' Track new virtual object (only if slug is set). '''
if vobj.slug:
self._data[vobj.url_path] = (vobj.group, vobj.config)

2
lektor_groupby/vobj.py
View File

@@ -158,7 +158,7 @@ class VGroups:
record: Record,
*keys: str,
recursive: bool = False
) -> Iterator['GroupBySource']:
) -> Iterator[GroupBySource]:
''' Extract all referencing groupby virtual objects from a page. '''
ctx = get_ctx()
# manage dependencies