v0.9.7

- postpone building until really needed
- rebuild only if artifacts change
- no build on source update
- prune takes current resolver state instead of global var

.gitignore

@@ -1,5 +1,5 @@
 .DS_Store
-/dist-env/
+/env-publish/
 
 __pycache__/
 *.py[cod]

Makefile

@@ -1,21 +1,16 @@
-.PHONY: help
+dist: setup.py lektor_groupby/*
-help:
-	@echo 'commands:'
-	@echo '  dist'
-
-dist-env:
-	@echo Creating virtual environment...
-	@python3 -m venv 'dist-env'
-	@source dist-env/bin/activate && pip install twine
-
-.PHONY: dist
-dist: dist-env
-	[ -z "$${VIRTUAL_ENV}" ]  # you can not do this inside a virtual environment.
-	rm -rf dist
 	@echo Building...
 	python3 setup.py sdist bdist_wheel
-	@echo
 	rm -rf ./*.egg-info/ ./build/ MANIFEST
+
+env-publish:
+	@echo Creating virtual environment...
+	@python3 -m venv 'env-publish'
+	@source env-publish/bin/activate && pip install twine
+
+.PHONY: publish
+publish: dist env-publish
+	[ -z "$${VIRTUAL_ENV}" ]  # you can not do this inside a virtual environment.
 	@echo Publishing...
-	@echo "\033[0;31mEnter your PyPI token:\033[0m"
+	@echo "\033[0;31mEnter PyPI token in password prompt:\033[0m"
-	@source dist-env/bin/activate && export TWINE_USERNAME='__token__' && twine upload dist/*
+	@source env-publish/bin/activate && export TWINE_USERNAME='__token__' && twine upload dist/*

examples/Examples.lektorproject

@@ -2,4 +2,4 @@
 name = GroupBy Examples
 
 [packages]
-lektor-groupby = 0.9.3
+lektor-groupby = 0.9.7

examples/README.md

@@ -6,12 +6,14 @@ Overview:
 - [advanced example](#advanced-example) touches on the potentials of the plugin.
 - [Misc](#misc) shows other use-cases.
 
+After reading this tutorial, have a look at other plugins that use `lektor-groupby`:
+- [lektor-inlinetags](https://github.com/relikd/lektor-inlinetags-plugin)
 
 
 ## About
 
 To use the groupby plugin you have to add an attribute to your model file.
-In our case you can refer to the `models/page.ini` model:
+In our case you can refer to the [`models/page.ini`](./models/page.ini) model:
 
 ```ini
 [fields.tags]
@@ -36,10 +38,10 @@ The attribute name is later used for grouping.
 ## Quick config
 
 Relevant files:
-```
+
-configs/groupby.ini
+- [`configs/groupby.ini`](./configs/groupby.ini)
-templates/example-config.html
+- [`templates/example-config.html`](./templates/example-config.html)
-```
+
 
 The easiest way to add tags to your site is by defining the `groupby.ini` config file.
 
@@ -133,10 +135,10 @@ In your template file you have access to the attributes, config, and children (p
 ## Simple example
 
 Relevant files:
-```
+
-packages/simple-example/lektor_simple.py
+- [`packages/simple-example/lektor_simple.py`](./packages/simple-example/lektor_simple.py)
-templates/example-simple.html
+- [`templates/example-simple.html`](./templates/example-simple.html)
-```
+
 
 ```python
 def on_groupby_before_build_all(self, groupby, builder, **extra):
@@ -204,11 +206,11 @@ The template file can access and display the `extra-info`:
 ## Advanced example
 
 Relevant files:
-```
+
-configs/advanced.ini
+- [`configs/advanced.ini`](./configs/advanced.ini)
-packages/advanced-example/lektor_advanced.py
+- [`packages/advanced-example/lektor_advanced.py`](./packages/advanced-example/lektor_advanced.py)
-templates/example-advanced.html
+- [`templates/example-advanced.html`](./templates/example-advanced.html)
-```
+
 
 The following example is similar to the previous one.
 Except that it loads a config file and replaces in-text occurrences of `{{Tagname}}` with `<a href="/tag/">Tagname</a>`.
@@ -277,11 +279,14 @@ This is useful if you do not want to create subpages but rather an index page co
 This can be done in combination with the next use-case:
 
 ```jinja2
-{%- for x in this|vgroups('TestA', 'TestB', recursive=True)|unique|sort %} 
+{%- for x in this|vgroups(keys=['TestA', 'TestB'], fields=[], flows=[], recursive=True)|unique|sort %} 
 <a href="{{ x|url }}">({{ x.group }})</a>
 {%- endfor %} 
 ```
 
 You can query the groups of any parent node (including those without slug).
+[`templates/page.html`](./templates/page.html) uses this.
 The keys (`'TestA', 'TestB'`) can be omitted which will return all groups of all attributes (you can still filter them with `x.config.key == 'TestC'`).
-Refer to `templates/page.html` for usage.
+The `fields` and `flows` params are also optional.
+With these you can match groups in `args.key.fieldKey` and `args.key.flowKey`.
+For example, if you have a “tags” field and an “additional-tags” field and you only want to show one in a preview.

lektor_groupby/backref.py

@@ -0,0 +1,83 @@
+from lektor.context import get_ctx
+from typing import TYPE_CHECKING, Union, Iterable, Iterator
+import weakref
+if TYPE_CHECKING:
+    from lektor.builder import Builder
+    from lektor.db import Record
+    from .groupby import GroupBy
+    from .model import FieldKeyPath
+    from .vobj import GroupBySource
+
+
+class WeakVGroupsList(list):
+    def add(self, strong: 'FieldKeyPath', weak: 'GroupBySource') -> None:
+        super().append((strong, weakref.ref(weak)))
+
+
+class GroupByRef:
+    @staticmethod
+    def of(builder: 'Builder') -> 'GroupBy':
+        ''' Get the GroupBy object of a builder. '''
+        return builder.__groupby  # type:ignore[attr-defined,no-any-return]
+
+    @staticmethod
+    def set(builder: 'Builder', groupby: 'GroupBy') -> None:
+        ''' Set the GroupBy object of a builder. '''
+        builder.__groupby = groupby  # type: ignore[attr-defined]
+
+
+class VGroups:
+    @staticmethod
+    def of(record: 'Record') -> WeakVGroupsList:
+        '''
+        Return the (weak) set of virtual objects of a page.
+        Creates a new set if it does not exist yet.
+        '''
+        try:
+            wset = record.__vgroups  # type: ignore[attr-defined]
+        except AttributeError:
+            wset = WeakVGroupsList()
+            record.__vgroups = wset  # type: ignore[attr-defined]
+        return wset  # type: ignore[no-any-return]
+
+    @staticmethod
+    def iter(
+        record: 'Record',
+        keys: Union[str, Iterable[str], None] = None,
+        *,
+        fields: Union[str, Iterable[str], None] = None,
+        flows: Union[str, Iterable[str], None] = None,
+        recursive: bool = False
+    ) -> Iterator['GroupBySource']:
+        ''' Extract all referencing groupby virtual objects from a page. '''
+        ctx = get_ctx()
+        if not ctx:
+            raise NotImplementedError("Shouldn't happen, where is my context?")
+        # get GroupBy object
+        builder = ctx.build_state.builder
+        groupby = GroupByRef.of(builder)
+        groupby.make_once(builder)  # ensure did cluster before
+        # manage config dependencies
+        for dep in groupby.dependencies:
+            ctx.record_dependency(dep)
+        # prepare filter
+        if isinstance(keys, str):
+            keys = [keys]
+        if isinstance(fields, str):
+            fields = [fields]
+        if isinstance(flows, str):
+            flows = [flows]
+        # find groups
+        proc_list = [record]
+        while proc_list:
+            page = proc_list.pop(0)
+            if recursive and hasattr(page, 'children'):
+                proc_list.extend(page.children)  # type: ignore[attr-defined]
+            for key, vobj in VGroups.of(page):
+                if fields and key.fieldKey not in fields:
+                    continue
+                if flows and key.flowKey not in flows:
+                    continue
+                if keys and vobj().config.key not in keys:
+                    continue
+                yield vobj()

lektor_groupby/groupby.py

@@ -1,12 +1,14 @@
-from lektor.builder import Builder, PathCache
+from lektor.builder import PathCache
-from lektor.db import Record
+from lektor.db import Record  # isinstance
-from lektor.sourceobj import SourceObject
+from typing import TYPE_CHECKING, Set, List
-from lektor.utils import build_url
+from .config import Config
-
-from typing import Set, Dict, List, Optional, Tuple
-from .vobj import GroupBySource
-from .config import Config, AnyConfig
 from .watcher import Watcher
+if TYPE_CHECKING:
+    from .config import AnyConfig
+    from lektor.builder import Builder
+    from lektor.sourceobj import SourceObject
+    from .resolver import Resolver
+    from .vobj import GroupBySource
 
 
 class GroupBy:
@@ -16,101 +18,63 @@ class GroupBy:
     The grouping is performed only once per build.
     '''
 
-    def __init__(self) -> None:
+    def __init__(self, resolver: 'Resolver') -> None:
         self._watcher = []  # type: List[Watcher]
         self._results = []  # type: List[GroupBySource]
-        self._resolver = {}  # type: Dict[str, Tuple[str, Config]]
+        self.resolver = resolver
 
-    # ----------------
+    def add_watcher(self, key: str, config: 'AnyConfig') -> Watcher:
-    #   Add observer
-    # ----------------
-
-    def add_watcher(self, key: str, config: AnyConfig) -> Watcher:
         ''' Init Config and add to watch list. '''
         w = Watcher(Config.from_any(key, config))
         self._watcher.append(w)
         return w
 
-    # -----------
-    #   Builder
-    # -----------
-
-    def clear_previous_results(self) -> None:
-        ''' Reset prvious results. Must be called before each build. '''
-        self._watcher.clear()
-        self._results.clear()
-        self._resolver.clear()
-
     def get_dependencies(self) -> Set[str]:
         deps = set()  # type: Set[str]
         for w in self._watcher:
             deps.update(w.config.dependencies)
         return deps
 
-    def make_cluster(self, builder: Builder) -> None:
+    def queue_all(self, builder: 'Builder') -> None:
-        ''' Iterate over all children and perform groupby. '''
+        ''' Iterate full site-tree and queue all children. '''
+        self.dependencies = self.get_dependencies()
         # remove disabled watchers
         self._watcher = [w for w in self._watcher if w.config.enabled]
         if not self._watcher:
             return
         # initialize remaining (enabled) watchers
         for w in self._watcher:
-            w.initialize(builder.pad.db)
+            w.initialize(builder.pad)
         # iterate over whole build tree
         queue = builder.pad.get_all_roots()  # type: List[SourceObject]
         while queue:
             record = queue.pop()
-            self.queue_now(record)
             if hasattr(record, 'attachments'):
                 queue.extend(record.attachments)  # type: ignore[attr-defined]
             if hasattr(record, 'children'):
                 queue.extend(record.children)  # type: ignore[attr-defined]
-        # build artifacts
+            if isinstance(record, Record):
-        for w in self._watcher:
+                for w in self._watcher:
-            root = builder.pad.get(w.config.root)
+                    if w.should_process(record):
-            for vobj in w.iter_sources(root):
+                        w.process(record)
-                self._results.append(vobj)
-                if vobj.slug:
-                    self._resolver[vobj.url_path] = (vobj.group, w.config)
-        self._watcher.clear()
 
-    def queue_now(self, node: SourceObject) -> None:
+    def make_once(self, builder: 'Builder') -> None:
-        ''' Process record immediatelly (No-Op if already processed). '''
+        ''' Perform groupby, iter over sources with watcher callback. '''
-        if isinstance(node, Record):
+        if self._watcher:
+            self.resolver.reset()
             for w in self._watcher:
-                if w.should_process(node):
+                root = builder.pad.get(w.config.root)
-                    w.process(node)
+                for vobj in w.iter_sources(root):
+                    self._results.append(vobj)
+                    self.resolver.add(vobj)
+            self._watcher.clear()
 
-    def build_all(self, builder: Builder) -> None:
+    def build_all(self, builder: 'Builder') -> None:
         ''' Create virtual objects and build sources. '''
+        self.make_once(builder)  # in case no page used the |vgroups filter
         path_cache = PathCache(builder.env)
         for vobj in self._results:
             if vobj.slug:
                 builder.build(vobj, path_cache)
         del path_cache
         self._results.clear()  # garbage collect weak refs
-
-    # -----------------
-    #   Path resolver
-    # -----------------
-
-    def resolve_dev_server_path(self, node: SourceObject, pieces: List[str]) \
-            -> Optional[GroupBySource]:
-        ''' Dev server only: Resolves path/ -> path/index.html '''
-        if isinstance(node, Record):
-            rv = self._resolver.get(build_url([node.url_path] + pieces))
-            if rv:
-                return GroupBySource(node, group=rv[0], config=rv[1])
-        return None
-
-    def resolve_virtual_path(self, node: SourceObject, pieces: List[str]) \
-            -> Optional[GroupBySource]:
-        ''' Admin UI only: Prevent server error and null-redirect. '''
-        if isinstance(node, Record) and len(pieces) >= 2:
-            path = node['_path']  # type: str
-            key, grp, *_ = pieces
-            for group, conf in self._resolver.values():
-                if key == conf.key and path == conf.root:
-                    if conf.slugify(group) == grp:
-                        return GroupBySource(node, group, conf)
-        return None

lektor_groupby/model.py

@@ -0,0 +1,66 @@
+from lektor.db import Database, Record  # typing
+from lektor.types.flow import Flow, FlowType
+from lektor.utils import bool_from_string
+from typing import Set, Dict, Tuple, Any, NamedTuple, Optional, Iterator
+
+
+class FieldKeyPath(NamedTuple):
+    fieldKey: str
+    flowIndex: Optional[int] = None
+    flowKey: Optional[str] = None
+
+
+class ModelReader:
+    '''
+    Find models and flow-models which contain attribute.
+    Flows are either returned directly (flatten=False) or
+    expanded so that each flow-block is yielded (flatten=True)
+    '''
+
+    def __init__(self, db: Database, attr: str, flatten: bool = False) -> None:
+        self.flatten = flatten
+        self._flows = {}  # type: Dict[str, Set[str]]
+        self._models = {}  # type: Dict[str, Dict[str, str]]
+        # find flow blocks containing attribute
+        for key, flow in db.flowblocks.items():
+            tmp1 = set(f.name for f in flow.fields
+                       if bool_from_string(f.options.get(attr, False)))
+            if tmp1:
+                self._flows[key] = tmp1
+        # find models and flow-blocks containing attribute
+        for key, model in db.datamodels.items():
+            tmp2 = {}  # Dict[str, str]
+            for field in model.fields:
+                if bool_from_string(field.options.get(attr, False)):
+                    tmp2[field.name] = '*'  # include all children
+                elif isinstance(field.type, FlowType) and self._flows:
+                    # only processed if at least one flow has attr
+                    fbs = field.type.flow_blocks
+                    # if fbs == None, all flow-blocks are allowed
+                    if fbs is None or any(x in self._flows for x in fbs):
+                        tmp2[field.name] = '?'  # only some flow blocks
+            if tmp2:
+                self._models[key] = tmp2
+
+    def read(self, record: Record) -> Iterator[Tuple[FieldKeyPath, Any]]:
+        ''' Enumerate all fields of a Record with attrib = True. '''
+        assert isinstance(record, Record)
+        for r_key, subs in self._models.get(record.datamodel.id, {}).items():
+            field = record[r_key]
+            if not field:
+                continue
+            if subs == '*':  # either normal field or flow type (all blocks)
+                if self.flatten and isinstance(field, Flow):
+                    for i, flow in enumerate(field.blocks):
+                        flowtype = flow['_flowblock']
+                        for f_key, block in flow._data.items():
+                            if f_key.startswith('_'):  # e.g., _flowblock
+                                continue
+                            yield FieldKeyPath(r_key, i, f_key), block
+                else:
+                    yield FieldKeyPath(r_key), field
+            else:  # always flow type (only some blocks)
+                for i, flow in enumerate(field.blocks):
+                    flowtype = flow['_flowblock']
+                    for f_key in self._flows.get(flowtype, []):
+                        yield FieldKeyPath(r_key, i, f_key), flow[f_key]

lektor_groupby/plugin.py

@@ -1,67 +1,81 @@
-from lektor.builder import Builder  # typing
+from lektor.db import Page  # isinstance
 from lektor.pluginsystem import Plugin  # subclass
-from lektor.sourceobj import SourceObject  # typing
+from typing import TYPE_CHECKING, Iterator, Any
-
+from .backref import GroupByRef, VGroups
-from typing import List, Optional, Iterator
-from .vobj import GroupBySource, GroupByBuildProgram, VPATH
 from .groupby import GroupBy
 from .pruner import prune
-from .watcher import GroupByCallbackArgs  # typing
+from .resolver import Resolver
+from .vobj import VPATH, GroupBySource, GroupByBuildProgram
+if TYPE_CHECKING:
+    from lektor.builder import Builder, BuildState
+    from lektor.sourceobj import SourceObject
+    from .watcher import GroupByCallbackArgs
 
 
 class GroupByPlugin(Plugin):
     name = 'GroupBy Plugin'
     description = 'Cluster arbitrary records with field attribute keyword.'
 
-    def on_setup_env(self, **extra: object) -> None:
+    def on_setup_env(self, **extra: Any) -> None:
-        self.creator = GroupBy()
+        self.has_changes = False
+        self.resolver = Resolver(self.env)
         self.env.add_build_program(GroupBySource, GroupByBuildProgram)
-        self.env.jinja_env.filters.update(vgroups=GroupBySource.of_record)
+        self.env.jinja_env.filters.update(vgroups=VGroups.iter)
 
-        # resolve /tag/rss/ -> /tag/rss/index.html (local server only)
+    def on_before_build(
-        @self.env.urlresolver
+        self, builder: 'Builder', source: 'SourceObject', **extra: Any
-        def a(node: SourceObject, parts: List[str]) -> Optional[GroupBySource]:
+    ) -> None:
-            return self.creator.resolve_dev_server_path(node, parts)
+        # before-build may be called before before-build-all (issue #1017)
+        # make sure it is always evaluated first
+        if isinstance(source, Page):
+            self._init_once(builder)
 
-        # resolve virtual objects in admin UI
+    def on_after_build(self, build_state: 'BuildState', **extra: Any) -> None:
-        @self.env.virtualpathresolver(VPATH.lstrip('@'))
+        if build_state.updated_artifacts:
-        def b(node: SourceObject, parts: List[str]) -> Optional[GroupBySource]:
+            self.has_changes = True
-            return self.creator.resolve_virtual_path(node, parts)
 
-    def _load_quick_config(self) -> None:
+    def on_after_build_all(self, builder: 'Builder', **extra: Any) -> None:
+        # only rebuild if has changes (bypass idle builds)
+        # or the very first time after startup (url resolver & pruning)
+        if self.has_changes or not self.resolver.has_any:
+            self._init_once(builder).build_all(builder)  # updates resolver
+            self.has_changes = False
+
+    def on_after_prune(self, builder: 'Builder', **extra: Any) -> None:
+        # TODO: find a better way to prune unreferenced elements
+        prune(builder, VPATH, self.resolver.files)
+
+    # ------------
+    #   internal
+    # ------------
+
+    def _init_once(self, builder: 'Builder') -> GroupBy:
+        try:
+            return GroupByRef.of(builder)
+        except AttributeError:
+            groupby = GroupBy(self.resolver)
+            GroupByRef.set(builder, groupby)
+
+        self._load_quick_config(groupby)
+        # let other plugins register their @groupby.watch functions
+        self.emit('before-build-all', groupby=groupby, builder=builder)
+        groupby.queue_all(builder)
+        return groupby
+
+    def _load_quick_config(self, groupby: GroupBy) -> None:
         ''' Load config file quick listeners. '''
         config = self.get_config()
         for key in config.sections():
             if '.' in key:  # e.g., key.fields and key.key_map
                 continue
 
-            watcher = self.creator.add_watcher(key, config)
+            watcher = groupby.add_watcher(key, config)
             split = config.get(key + '.split')  # type: str
 
             @watcher.grouping()
-            def _fn(args: GroupByCallbackArgs) -> Iterator[str]:
+            def _fn(args: 'GroupByCallbackArgs') -> Iterator[str]:
                 val = args.field
                 if isinstance(val, str):
-                    val = val.split(split) if split else [val]  # make list
+                    val = map(str.strip, val.split(split)) if split else [val]
-                if isinstance(val, list):
+                if isinstance(val, (list, map)):
                     yield from val
-
-    def on_before_build_all(self, builder: Builder, **extra: object) -> None:
-        self.creator.clear_previous_results()
-        self._load_quick_config()
-        # let other plugins register their @groupby.watch functions
-        self.emit('before-build-all', groupby=self.creator, builder=builder)
-        self.config_dependencies = self.creator.get_dependencies()
-        self.creator.make_cluster(builder)
-
-    def on_before_build(self, source: SourceObject, **extra: object) -> None:
-        # before-build may be called before before-build-all (issue #1017)
-        # make sure it is evaluated immediatelly
-        self.creator.queue_now(source)
-
-    def on_after_build_all(self, builder: Builder, **extra: object) -> None:
-        self.creator.build_all(builder)
-
-    def on_after_prune(self, builder: Builder, **extra: object) -> None:
-        # TODO: find a better way to prune unreferenced elements
-        prune(builder, VPATH)

lektor_groupby/pruner.py

@@ -2,29 +2,36 @@
 Static collector for build-artifact urls.
 All non-tracked VPATH-urls will be pruned after build.
 '''
-from lektor.builder import Builder  # typing
 from lektor.reporter import reporter  # report_pruned_artifact
 from lektor.utils import prune_file_and_folder
-
+from typing import TYPE_CHECKING, Set, Iterable
-_cache = set()
+if TYPE_CHECKING:
-# Note: this var is static or otherwise two instances of
+    from lektor.builder import Builder
-#       this module would prune each others artifacts.
 
 
-def track_not_prune(url: str) -> None:
+def _normalize_url_cache(url_cache: Iterable[str]) -> Set[str]:
-    ''' Add url to build cache to prevent pruning. '''
+    cache = set()
-    _cache.add(url.lstrip('/'))
+    for url in url_cache:
+        if url.endswith('/'):
+            url += 'index.html'
+        cache.add(url.lstrip('/'))
+    return cache
 
 
-def prune(builder: Builder, vpath: str) -> None:
+def prune(builder: 'Builder', vpath: str, url_cache: Iterable[str]) -> None:
-    ''' Remove previously generated, unreferenced Artifacts. '''
+    '''
+    Remove previously generated, unreferenced Artifacts.
+    All urls in url_cache must have a trailing "/index.html" (instead of "/")
+    and also, no leading slash, "blog/index.html" instead of "/blog/index.html"
+    '''
     vpath = '@' + vpath.lstrip('@')  # just in case of user error
     dest_path = builder.destination_path
+    url_cache = _normalize_url_cache(url_cache)
     con = builder.connect_to_database()
     try:
         with builder.new_build_state() as build_state:
             for url, file in build_state.iter_artifacts():
-                if url.lstrip('/') in _cache:
+                if url.lstrip('/') in url_cache:
                     continue  # generated in this build-run
                 infos = build_state.get_artifact_dependency_infos(url, [])
                 for artifact_name, _ in infos:
@@ -36,4 +43,3 @@ def prune(builder: Builder, vpath: str) -> None:
                     break  # there is only one VPATH-entry per source
     finally:
         con.close()
-    _cache.clear()

lektor_groupby/resolver.py

@@ -0,0 +1,61 @@
+from lektor.db import Record  # isinstance
+from lektor.utils import build_url
+from typing import TYPE_CHECKING, Dict, List, Tuple, Optional, Iterable
+from .vobj import VPATH, GroupBySource
+if TYPE_CHECKING:
+    from lektor.environment import Environment
+    from lektor.sourceobj import SourceObject
+    from .config import Config
+
+
+class Resolver:
+    '''
+    Resolve virtual paths and urls ending in /.
+    Init will subscribe to @urlresolver and @virtualpathresolver.
+    '''
+
+    def __init__(self, env: 'Environment') -> None:
+        self._data = {}  # type: Dict[str, Tuple[str, str, Config]]
+        env.urlresolver(self.resolve_server_path)
+        env.virtualpathresolver(VPATH.lstrip('@'))(self.resolve_virtual_path)
+
+    @property
+    def has_any(self) -> bool:
+        return bool(self._data)
+
+    @property
+    def files(self) -> Iterable[str]:
+        return self._data
+
+    def reset(self) -> None:
+        ''' Clear previously recorded virtual objects. '''
+        self._data.clear()
+
+    def add(self, vobj: GroupBySource) -> None:
+        ''' Track new virtual object (only if slug is set). '''
+        if vobj.slug:
+            self._data[vobj.url_path] = (vobj.key, vobj.group, vobj.config)
+
+    # ------------
+    #   Resolver
+    # ------------
+
+    def resolve_server_path(self, node: 'SourceObject', pieces: List[str]) \
+            -> Optional[GroupBySource]:
+        ''' Local server only: resolve /tag/rss/ -> /tag/rss/index.html '''
+        if isinstance(node, Record):
+            rv = self._data.get(build_url([node.url_path] + pieces))
+            if rv:
+                return GroupBySource(node, rv[0]).finalize(rv[2], rv[1])
+        return None
+
+    def resolve_virtual_path(self, node: 'SourceObject', pieces: List[str]) \
+            -> Optional[GroupBySource]:
+        ''' Admin UI only: Prevent server error and null-redirect. '''
+        if isinstance(node, Record) and len(pieces) >= 2:
+            path = node['_path']  # type: str
+            attr, grp, *_ = pieces
+            for slug, group, conf in self._data.values():
+                if attr == conf.key and slug == grp and path == conf.root:
+                    return GroupBySource(node, slug).finalize(conf, group)
+        return None

lektor_groupby/util.py

@@ -1,5 +1,7 @@
 from lektor.reporter import reporter, style
 
+from typing import List, Dict
+
 
 def report_config_error(key: str, field: str, val: str, e: Exception) -> None:
     ''' Send error message to Lektor reporter. Indicate which field is bad. '''
@@ -9,3 +11,18 @@ def report_config_error(key: str, field: str, val: str, e: Exception) -> None:
         reporter._write_line(style(msg, fg='red'))
     except Exception:
         print(msg)  # fallback in case Lektor API changes
+
+
+def most_used_key(keys: List[str]) -> str:
+    if len(keys) < 3:
+        return keys[0]  # TODO: first vs last occurrence
+    best_count = 0
+    best_key = ''
+    tmp = {}  # type: Dict[str, int]
+    for k in keys:
+        num = (tmp[k] + 1) if k in tmp else 1
+        tmp[k] = num
+        if num > best_count:  # TODO: (>) vs (>=), first vs last occurrence
+            best_count = num
+            best_key = k
+    return best_key

lektor_groupby/vobj.py

@@ -1,16 +1,14 @@
 from lektor.build_programs import BuildProgram  # subclass
-from lektor.builder import Artifact  # typing
 from lektor.context import get_ctx
-from lektor.db import Record  # typing
 from lektor.environment import Expression
 from lektor.sourceobj import VirtualSourceObject  # subclass
 from lektor.utils import build_url
-
+from typing import TYPE_CHECKING, Dict, List, Any, Optional, Iterator
-from typing import Dict, List, Any, Optional, Iterator
+from .util import report_config_error, most_used_key
-from weakref import WeakSet
+if TYPE_CHECKING:
-from .config import Config
+    from lektor.builder import Artifact
-from .pruner import track_not_prune
+    from lektor.db import Record
-from .util import report_config_error
+    from .config import Config
 
 VPATH = '@groupby'  # potentially unsafe. All matching entries are pruned.
 
@@ -26,17 +24,29 @@ class GroupBySource(VirtualSourceObject):
     Attributes: record, key, group, slug, children, config
     '''
 
-    def __init__(
+    def __init__(self, record: 'Record', slug: str) -> None:
-        self,
-        record: Record,
-        group: str,
-        config: Config,
-        children: Optional[Dict[Record, List[Any]]] = None,
-    ) -> None:
         super().__init__(record)
-        self.key = config.slugify(group)
+        self.key = slug
-        self.group = group
+        self._group_map = []  # type: List[str]
+        self._children = {}  # type: Dict[Record, List[Any]]
+
+    def append_child(self, child: 'Record', extra: Any, group: str) -> None:
+        if child not in self._children:
+            self._children[child] = [extra]
+        else:
+            self._children[child].append(extra)
+        # _group_map is later used to find most used group
+        self._group_map.append(group)
+
+    # -------------------------
+    #   Evaluate Extra Fields
+    # -------------------------
+
+    def finalize(self, config: 'Config', group: Optional[str] = None) \
+            -> 'GroupBySource':
         self.config = config
+        self.group = group or most_used_key(self._group_map)
+        del self._group_map
         # evaluate slug Expression
         if config.slug and '{key}' in config.slug:
             self.slug = config.slug.replace('{key}', self.key)
@@ -45,16 +55,10 @@ class GroupBySource(VirtualSourceObject):
             assert self.slug != Ellipsis, 'invalid config: ' + config.slug
         if self.slug and self.slug.endswith('/index.html'):
             self.slug = self.slug[:-10]
-        # make sure children are on the same pad
-        self._children = {}  # type: Dict[Record, List[Any]]
-        for child, extras in (children or {}).items():
-            if child.pad != record.pad:
-                child = record.pad.get(child.path)
-            self._children[child] = extras
-        self._reverse_reference_records()
         # extra fields
         for attr, expr in config.fields.items():
             setattr(self, attr, self._eval(expr, field='fields.' + attr))
+        return self
 
     def _eval(self, value: Any, *, field: str) -> Any:
         ''' Internal only: evaluates Lektor config file field expression. '''
@@ -94,12 +98,12 @@ class GroupBySource(VirtualSourceObject):
     # -----------------------
 
     @property
-    def children(self) -> Dict[Record, List[Any]]:
+    def children(self) -> Dict['Record', List[Any]]:
         ''' Returns dict with page record key and (optional) extra value. '''
         return self._children
 
     @property
-    def first_child(self) -> Optional[Record]:
+    def first_child(self) -> Optional['Record']:
         ''' Returns first referencing page record. '''
         if self._children:
             return iter(self._children).__next__()
@@ -114,16 +118,14 @@ class GroupBySource(VirtualSourceObject):
         return val[0] if val else None
 
     def __getitem__(self, key: str) -> Any:
-        # Used for virtual path resolver and |sort(attribute="x") filter
+        # Used for virtual path resolver
         if key in ('_path', '_alt'):
             return getattr(self, key[1:])
-        if hasattr(self, key):
+        return self.__missing__(key)  # type: ignore[attr-defined]
-            return getattr(self, key)
-        return None
 
     def __lt__(self, other: 'GroupBySource') -> bool:
         # Used for |sort filter ("group" is the provided original string)
-        return self.group < other.group
+        return self.group.lower() < other.group.lower()
 
     def __eq__(self, other: object) -> bool:
         # Used for |unique filter
@@ -140,41 +142,6 @@ class GroupBySource(VirtualSourceObject):
         return '<GroupBySource path="{}" children={}>'.format(
             self.path, len(self._children))
 
-    # ---------------------
-    #   Reverse Reference
-    # ---------------------
-
-    def _reverse_reference_records(self) -> None:
-        ''' Attach self to page records. '''
-        for child in self._children:
-            if not hasattr(child, '_vgroups'):
-                child._vgroups = WeakSet()  # type: ignore[attr-defined]
-            child._vgroups.add(self)  # type: ignore[attr-defined]
-
-    @staticmethod
-    def of_record(
-        record: Record,
-        *keys: str,
-        recursive: bool = False
-    ) -> Iterator['GroupBySource']:
-        ''' Extract all referencing groupby virtual objects from a page. '''
-        ctx = get_ctx()
-        # manage dependencies
-        if ctx:
-            for dep in ctx.env.plugins['groupby'].config_dependencies:
-                ctx.record_dependency(dep)
-        # find groups
-        proc_list = [record]
-        while proc_list:
-            page = proc_list.pop(0)
-            if recursive and hasattr(page, 'children'):
-                proc_list.extend(page.children)  # type: ignore[attr-defined]
-            if not hasattr(page, '_vgroups'):
-                continue
-            for vobj in page._vgroups:  # type: ignore[attr-defined]
-                if not keys or vobj.config.key in keys:
-                    yield vobj
-
 
 # -----------------------------------
 #           BuildProgram
@@ -189,9 +156,8 @@ class GroupByBuildProgram(BuildProgram):
             url += 'index.html'
         self.declare_artifact(url, sources=list(
             self.source.iter_source_filenames()))
-        track_not_prune(url)
 
-    def build_artifact(self, artifact: Artifact) -> None:
+    def build_artifact(self, artifact: 'Artifact') -> None:
         get_ctx().record_virtual_dependency(self.source)
         artifact.render_template_into(
             self.source.config.template, this=self.source)

lektor_groupby/watcher.py

@@ -1,26 +1,17 @@
-from lektor.db import Database, Record  # typing
+from typing import TYPE_CHECKING, Dict, List, Tuple, Any, Union, NamedTuple
-from lektor.types.flow import Flow, FlowType
+from typing import Optional, Callable, Iterator, Generator
-from lektor.utils import bool_from_string
+from .backref import VGroups
-
+from .model import ModelReader
-from typing import Set, Dict, List, Tuple, Any, Union, NamedTuple
-from typing import Optional, Callable, Iterable, Iterator, Generator
 from .vobj import GroupBySource
-from .config import Config
+if TYPE_CHECKING:
-
+    from lektor.db import Pad, Record
-
+    from .config import Config
-# -----------------------------------
+    from .model import FieldKeyPath
-#              Typing
-# -----------------------------------
-
-class FieldKeyPath(NamedTuple):
-    fieldKey: str
-    flowIndex: Optional[int] = None
-    flowKey: Optional[str] = None
 
 
 class GroupByCallbackArgs(NamedTuple):
-    record: Record
+    record: 'Record'
-    key: FieldKeyPath
+    key: 'FieldKeyPath'
     field: Any  # lektor model data-field value
 
 
@@ -30,111 +21,15 @@ GroupingCallback = Callable[[GroupByCallbackArgs], Union[
 ]]
 
 
-# -----------------------------------
-#            ModelReader
-# -----------------------------------
-
-class GroupByModelReader:
-    ''' Find models and flow-models which contain attribute '''
-
-    def __init__(self, db: Database, attrib: str) -> None:
-        self._flows = {}  # type: Dict[str, Set[str]]
-        self._models = {}  # type: Dict[str, Dict[str, str]]
-        # find flow blocks containing attribute
-        for key, flow in db.flowblocks.items():
-            tmp1 = set(f.name for f in flow.fields
-                       if bool_from_string(f.options.get(attrib, False)))
-            if tmp1:
-                self._flows[key] = tmp1
-        # find models and flow-blocks containing attribute
-        for key, model in db.datamodels.items():
-            tmp2 = {}  # Dict[str, str]
-            for field in model.fields:
-                if bool_from_string(field.options.get(attrib, False)):
-                    tmp2[field.name] = '*'  # include all children
-                elif isinstance(field.type, FlowType) and self._flows:
-                    # only processed if at least one flow has attrib
-                    fbs = field.type.flow_blocks
-                    # if fbs == None, all flow-blocks are allowed
-                    if fbs is None or any(x in self._flows for x in fbs):
-                        tmp2[field.name] = '?'  # only some flow blocks
-            if tmp2:
-                self._models[key] = tmp2
-
-    def read(
-        self,
-        record: Record,
-        flatten: bool = False
-    ) -> Iterator[Tuple[FieldKeyPath, Any]]:
-        '''
-        Enumerate all fields of a Record with attrib = True.
-        Flows are either returned directly (flatten=False) or
-        expanded so that each flow-block is yielded (flatten=True)
-        '''
-        assert isinstance(record, Record)
-        for r_key, subs in self._models.get(record.datamodel.id, {}).items():
-            if subs == '*':  # either normal field or flow type (all blocks)
-                field = record[r_key]
-                if flatten and isinstance(field, Flow):
-                    for i, flow in enumerate(field.blocks):
-                        flowtype = flow['_flowblock']
-                        for f_key, block in flow._data.items():
-                            if f_key.startswith('_'):  # e.g., _flowblock
-                                continue
-                            yield FieldKeyPath(r_key, i, f_key), block
-                else:
-                    yield FieldKeyPath(r_key), field
-            else:  # always flow type (only some blocks)
-                for i, flow in enumerate(record[r_key].blocks):
-                    flowtype = flow['_flowblock']
-                    for f_key in self._flows.get(flowtype, []):
-                        yield FieldKeyPath(r_key, i, f_key), flow[f_key]
-
-
-# -----------------------------------
-#               State
-# -----------------------------------
-
-class GroupByState:
-    ''' Store and update a groupby build state. {group: {record: [extras]}} '''
-
-    def __init__(self) -> None:
-        self.state = {}  # type: Dict[str, Dict[Record, List[Any]]]
-        self._processed = set()  # type: Set[Record]
-
-    def __contains__(self, record: Record) -> bool:
-        ''' Returns True if record was already processed. '''
-        return record.path in self._processed
-
-    def items(self) -> Iterable[Tuple[str, Dict[Record, List[Any]]]]:
-        ''' Iterable with (group, {record: [extras]}) tuples. '''
-        return self.state.items()
-
-    def add(self, record: Record, sub_groups: Dict[str, List[Any]]) -> None:
-        ''' Append groups if not processed already. {group: [extras]} '''
-        if record.path not in self._processed:
-            self._processed.add(record.path)
-            for group, extras in sub_groups.items():
-                if group in self.state:
-                    self.state[group][record] = extras
-                else:
-                    self.state[group] = {record: extras}
-
-
-# -----------------------------------
-#              Watcher
-# -----------------------------------
-
 class Watcher:
     '''
     Callback is called with (Record, FieldKeyPath, field-value).
     Callback may yield one or more (group, extra-info) tuples.
     '''
 
-    def __init__(self, config: Config) -> None:
+    def __init__(self, config: 'Config') -> None:
         self.config = config
-        self.flatten = True
+        self._root = self.config.root
-        self.callback = None  # type: GroupingCallback #type:ignore[assignment]
 
     def grouping(self, flatten: bool = True) \
             -> Callable[[GroupingCallback], None]:
@@ -149,50 +44,67 @@ class Watcher:
             self.callback = fn
         return _decorator
 
-    def initialize(self, db: Database) -> None:
+    def initialize(self, pad: 'Pad') -> None:
         ''' Reset internal state. You must initialize before each build! '''
         assert callable(self.callback), 'No grouping callback provided.'
-        self._root = self.config.root
+        self._model_reader = ModelReader(pad.db, self.config.key, self.flatten)
-        self._state = GroupByState()
+        self._root_record = pad.get(self._root)  # type: Record
-        self._model_reader = GroupByModelReader(db, attrib=self.config.key)
+        self._state = {}  # type: Dict[str, GroupBySource]
 
-    def should_process(self, node: Record) -> bool:
+    def should_process(self, node: 'Record') -> bool:
         ''' Check if record path is being watched. '''
         return node['_path'].startswith(self._root)
 
-    def process(self, record: Record) -> None:
+    def process(self, record: 'Record') -> None:
         '''
         Will iterate over all record fields and call the callback method.
         Each record is guaranteed to be processed only once.
         '''
-        if record in self._state:
+        for key, field in self._model_reader.read(record):
-            return
-        tmp = {}  # type: Dict[str, List[Any]] # {group: [extras]}
-        for key, field in self._model_reader.read(record, self.flatten):
             _gen = self.callback(GroupByCallbackArgs(record, key, field))
             try:
                 obj = next(_gen)
                 while True:
                     if not isinstance(obj, (str, tuple)):
                         raise TypeError(f'Unsupported groupby yield: {obj}')
-                    group = obj if isinstance(obj, str) else obj[0]
+                    slug = self._persist(record, key, obj)
-                    if group not in tmp:
-                        tmp[group] = []
-                    if isinstance(obj, tuple):
-                        tmp[group].append(obj[1])
                     # return slugified group key and continue iteration
                     if isinstance(_gen, Generator) and not _gen.gi_yieldfrom:
-                        obj = _gen.send(self.config.slugify(group))
+                        obj = _gen.send(slug)
                     else:
                         obj = next(_gen)
             except StopIteration:
                 del _gen
-        self._state.add(record, tmp)
 
-    def iter_sources(self, root: Record) -> Iterator[GroupBySource]:
+    def _persist(
+        self, record: 'Record', key: 'FieldKeyPath', obj: Union[str, tuple]
+    ) -> str:
+        ''' Update internal state. Return slugified string. '''
+        if isinstance(obj, str):
+            group, extra = obj, key.fieldKey
+        else:
+            group, extra = obj
+
+        slug = self.config.slugify(group)
+        if slug not in self._state:
+            src = GroupBySource(self._root_record, slug)
+            self._state[slug] = src
+        else:
+            src = self._state[slug]
+
+        src.append_child(record, extra, group)
+        # reverse reference
+        VGroups.of(record).add(key, src)
+        return slug
+
+    def iter_sources(self, root: 'Record') -> Iterator[GroupBySource]:
         ''' Prepare and yield GroupBySource elements. '''
-        for group, children in self._state.items():
+        for vobj in self._state.values():
-            yield GroupBySource(root, group, self.config, children=children)
+            yield vobj.finalize(self.config)
+        # cleanup. remove this code if you'd like to iter twice
+        del self._model_reader
+        del self._root_record
+        del self._state
 
     def __repr__(self) -> str:
         return '<GroupByWatcher key="{}" enabled={} callback={}>'.format(

setup.py

@@ -5,7 +5,7 @@ with open('README.md') as fp:
 
 setup(
     name='lektor-groupby',
-    py_modules=['lektor_groupby'],
+    packages=['lektor_groupby'],
     entry_points={
         'lektor.plugins': [
             'groupby = lektor_groupby:GroupByPlugin',
@@ -13,7 +13,7 @@ setup(
     },
     author='relikd',
     url='https://github.com/relikd/lektor-groupby-plugin',
-    version='0.9.3',
+    version='0.9.7',
     description='Cluster arbitrary records with field attribute keyword.',
     long_description=longdesc,
     long_description_content_type="text/markdown",
@@ -27,7 +27,6 @@ setup(
         'cluster',
     ],
     classifiers=[
-        'Development Status :: 5 - Production/Stable',
         'Environment :: Web Environment',
         'Environment :: Plugins',
         'Framework :: Lektor',