chore: bump v1.0.0

.gitignore

@@ -1,5 +1,5 @@
 .DS_Store
-/dist-env/
+/env-publish/
 
 __pycache__/
 *.py[cod]

CHANGELOG.md

@@ -0,0 +1,178 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/0.9.8/),
+and this project does adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+
+## [Unreleased]
+
+
+## [1.0.0] – 2023-03-04
+
+### Added
+- `vgroups` filter now supports `unique=False` to return a list of all entries including duplicates (default: `True`)
+
+### Fixed
+- Preserves original sort order in `vgroups` filter if `unique=True`
+- Remove duplicates from `GroupBySource` children
+
+### Changed
+- `vgroups` filter uses `recursive=True` by default
+
+
+
+## [0.9.9] – 2022-12-21
+
+### Fixed
+- Keep original sorting order in `vgroups` filter if no `order_by` is set
+
+
+
+## [0.9.8] – 2022-12-20
+
+### Added
+- Support for Alternatives
+- Support for Pagination
+- Support for additional yield types (str, int, float, bool)
+- Support for sorting `GroupBySource` children
+- Support for sorting `vgroups` filter
+- Config option `.replace_none_key` to replace `None` with another value
+- Config option `.key_obj_fn` (function) can be used to map complex objects to simple values (e.g., list of strings -> count as int). In your jinja template you may use `X` (the object) and `ARGS` (the `GroupByCallbackArgs`).
+- New property `supports_pagination` (bool) for `GroupBySource`
+- Partial building. Only process `Watcher` which are used during template rendering.
+- Rebuild `GroupBySource` only once after a `Record` update
+
+### Changed
+- Use `Query` for children instead of `Record` list
+- Rename `GroupBySource.group` to `GroupBySource.key_obj`
+- Yield return `GroupBySource` during `watcher.grouping()` instead of slugified key
+- Postpone `Record` processing until `make_once()`
+- Allow preprocessing with `pre_build=True` as optional parameter for `groupby.add_watcher()` (useful for modifying source before build)
+- Evaluate `fields` attributes upon access, not initialization (this comes with a more fine-grained dependency tracking)
+- Resolver groups virtual pages per groupby config key (before it was just a list of all groupby sources mixed together)
+- Refactor pruning by adding a `VirtualPruner` vobj
+- Pruning is performed directly on the database
+- `GroupBySource.path` may include a page number suffix `/2`
+- `GroupBySource.url_path` may include a page number and custom `url_suffix`
+
+### Removed
+- `GroupingCallback` may no longer yield an extra object. The usage was cumbersome and can be replaced with the `.fields` config option.
+
+### Fixed
+- `GroupBySource` not updated on template edit
+- `most_used_key` with empty list
+- Don't throw exception if `GroupBySource` is printed before finalize
+- Hotfix for Lektor issue #1085 by avoiding `TypeError`
+- Add missing dependencies during `vgroups` filter
+- Include model-fields with null value on yield
+
+
+
+## [0.9.7] – 2022-04-22
+
+### Changed
+- Refactor `GroupBySource` init method
+- Decouple `fields` expression processing from init
+
+### Fixed
+- Keep order of groups intact
+
+
+
+## [0.9.6] – 2022-04-13
+
+### Added
+- Set extra-info default to the model-key that generated the group.
+- Reuse previously declared `fields` attributes in later `fields`.
+
+### Changed
+- Thread-safe building. Each groupby is performed on the builder which initiated the build.
+- Deferred building. The groupby callback is only called when it is accessed for the first time.
+- Build-on-access. If there are no changes, no groupby build is performed.
+
+### Fixed
+- Inconsistent behavior due to concurrent building (see above)
+- Case insensitive default group sort
+- Using the split config-option now trims whitespace
+- `most_used_key` working properly
+
+
+
+## [0.9.5] – 2022-04-07
+
+### Fixed
+- Allow model instances without flow-blocks
+
+
+
+## [0.9.4] – 2022-04-06
+
+### Fixed
+- Error handling for GroupBySource `__getitem__` by raising `__missing__`
+- Reuse GroupBySource if two group names result in the same slug
+
+
+
+## [0.9.3] – 2022-04-06
+
+### Added
+- Config option `.fields` can add arbitrary attributes to the groupby
+- Config option `.key_map` allows to replace keys with other values (e.g., "C#" -> "C-Sharp")
+- Set `slug = None` to prevent rendering of groupby pages
+- Query groupby of children
+
+### Changed
+- Another full refactoring, constantly changing, everything is different ... again
+
+
+
+## [0.9.2] – 2022-04-01
+
+### Fixed
+- Prevent duplicate processing of records
+
+
+
+## [0.9.1] – 2022-03-31
+
+### Added
+- Example project
+- Before- and after-init hooks
+- More type hints (incl. bugfixes)
+
+### Changed
+- Encapsulate logic into separate classes
+
+### Fixed
+- Concurrency issues by complete refactoring
+- Virtual path and remove virtual path resolver
+
+
+
+## [0.9] – 2022-03-27
+
+### Fixed
+- Groupby is now generated before main page
+- PyPi readme
+
+
+
+## [0.8] – 2022-03-25
+
+Initial release
+
+
+[Unreleased]: https://github.com/relikd/lektor-groupby-plugin/compare/v1.0.0...HEAD
+[1.0.0]: https://github.com/relikd/lektor-groupby-plugin/compare/v0.9.9...v1.0.0
+[0.9.9]: https://github.com/relikd/lektor-groupby-plugin/compare/v0.9.8...v0.9.9
+[0.9.8]: https://github.com/relikd/lektor-groupby-plugin/compare/v0.9.7...v0.9.8
+[0.9.7]: https://github.com/relikd/lektor-groupby-plugin/compare/v0.9.6...v0.9.7
+[0.9.6]: https://github.com/relikd/lektor-groupby-plugin/compare/v0.9.5...v0.9.6
+[0.9.5]: https://github.com/relikd/lektor-groupby-plugin/compare/v0.9.4...v0.9.5
+[0.9.4]: https://github.com/relikd/lektor-groupby-plugin/compare/v0.9.3...v0.9.4
+[0.9.3]: https://github.com/relikd/lektor-groupby-plugin/compare/v0.9.2...v0.9.3
+[0.9.2]: https://github.com/relikd/lektor-groupby-plugin/compare/v0.9.1...v0.9.2
+[0.9.1]: https://github.com/relikd/lektor-groupby-plugin/compare/v0.9...v0.9.1
+[0.9]: https://github.com/relikd/lektor-groupby-plugin/compare/v0.8...v0.9
+[0.8]: https://github.com/relikd/lektor-groupby-plugin/releases/tag/v0.8

Makefile

@@ -1,4 +1,5 @@
 dist: setup.py lektor_groupby/*
+	[ -z "$${VIRTUAL_ENV}" ]  # you can not do this inside a virtual environment.
 	@echo Building...
 	python3 setup.py sdist bdist_wheel
 	rm -rf ./*.egg-info/ ./build/ MANIFEST

README.md

@@ -3,7 +3,6 @@
 A generic grouping / clustering plugin.
 Can be used for tagging or similar tasks.
 The grouping algorithm is performed once.
-Contrary to, at least, cubic runtime if doing the same with Pad queries.
 
 Install this plugin or modify your Lektor project file:
 
@@ -16,7 +15,7 @@ Optionally, enable a basic config:
 ```ini
 [tags]
 root = /
-slug = tag/{group}.html
+slug = tag/{key}.html
 template = tag.html
 split = ' '
 ```

examples/Examples.lektorproject

@@ -2,4 +2,4 @@
 name = GroupBy Examples
 
 [packages]
-lektor-groupby = 0.9.6
+lektor-groupby = 0.9.8

examples/README.md

@@ -1,19 +1,20 @@
 # Usage
 
 Overview:
-- [quick config example](#quick-config) shows how you can use the plugin config to setup a quick and easy tagging system.
+- [config example](#config-file) shows how you can use the plugin config to setup a quick and easy tagging system.
 - [simple example](#simple-example) goes into detail how to use it in your own plugin.
-- [advanced example](#advanced-example) touches on the potentials of the plugin.
+- [advanced example](#advanced-example) touches on the potentials of plugin development.
 - [Misc](#misc) shows other use-cases.
 
 After reading this tutorial, have a look at other plugins that use `lektor-groupby`:
 - [lektor-inlinetags](https://github.com/relikd/lektor-inlinetags-plugin)
 
 
+
 ## About
 
 To use the groupby plugin you have to add an attribute to your model file.
-In our case you can refer to the [`models/page.ini`](./models/page.ini) model:
+For this tutorial you can refer to the [`models/page.ini`](./models/page.ini) model:
 
 ```ini
 [fields.tags]
@@ -28,22 +29,16 @@ type = markdown
 testC = true
 ```
 
-We did define three custom attributes `testA`, `testB`, and `testC`.
+We define three custom attributes `testA`, `testB`, and `testC`.
 You may add custom attributes to all of the fields.
 It is crucial that the value of the custom attribute is set to true.
 The attribute name is later used for grouping.
 
 
 
-## Quick config
+## Config File
 
-Relevant files:
+The easiest way to add tags to your site is by defining the [`configs/groupby.ini`](./configs/groupby.ini) file.
-
-- [`configs/groupby.ini`](./configs/groupby.ini)
-- [`templates/example-config.html`](./templates/example-config.html)
-
-
-The easiest way to add tags to your site is by defining the `groupby.ini` config file.
 
 ```ini
 [testA]
@@ -52,14 +47,27 @@ slug = config/{key}.html
 template = example-config.html
 split = ' '
 enabled = True
+key_obj_fn = (X.upper() ~ ARGS.key.fieldKey) if X else 'empty'
+replace_none_key = unknown
+
+[testA.children]
+order_by = -title, body
+
+[testA.pagination]
+enabled = true
+per_page = 5
+url_suffix = .page.
 
 [testA.fields]
-title = "Tagged: " ~ this.group
+title = "Tagged: " ~ this.key_obj
 
 [testA.key_map]
-Blog = News
+C# = c-sharp
 ```
 
+
+### Config: Main Section
+
 The configuration parameter are:
 
 1. The section title (`testA`) must be one of the attribute variables we defined in our model.
@@ -81,46 +89,76 @@ The configuration parameter are:
    These single-line fields are then expanded to lists as well.
    If you do not provide the `split` option, the whole field value will be used as tagname.
 6. The `enabled` parameter allows you to quickly disable the grouping.
+7. The `key_obj_fn` parameter (jinja) accepts any function-like snippet or function call.
+   The context provides two variables, `X` and `ARGS`.
+   The former is the raw value of the grouping, this may be a text field, markdown, or whatever custom type you have provided.
+   The latter is a named tuple with `record`, `key`, and `field` values (see [simple example](#simple-example)).
+8. The `replace_none_key` parameter (string) is applied after `key_obj_fn` (if provided) and maps empty values to a default value.
 
 
 You can have multiple listeners, e.g., one for `/blog/` and another for `/projects/`.
-Just create as many custom attributes as you like, each having its own section.
+Just create as many custom attributes as you like, each having its own section (and subsections).
 
-There are two additional config mappings, `.fields` and `.key_map`.
+
-Key-value pairs in `.fields` will be added as attributes to your grouping.
+### Config Subsection: .children
+
+The `.children` subsection currently has a single config field: `order_by`.
+The usual [order-by](https://www.getlektor.com/docs/guides/page-order/) rules apply (comma separated list of keys with `-` for reversed order).
+The order-by key can be anything of the page attributes of the children.
+
+
+### Config Subsection: .pagination
+
+The `.pagination` subsection accepts the same configuration options as the default Lektor pagination ([model](https://www.getlektor.com/docs/models/children/#pagination), [guide](https://www.getlektor.com/docs/guides/pagination/)).
+Plus, an additional `url_suffix` parameter if you would like to customize the URL scheme.
+
+
+### Config Subsection: .fields
+
+The `.fields` subsection is a list of key-value pairs which will be added as attributes to your grouping.
 You can access them in your template (e.g., `{{this.title}}`).
-All of the `.fields` values are evaluted in a jinja context, so be cautious when using plain strings. 
+All of the `.fields` values are evaluted in a jinja context, so be cautious when using plain strings.
+Further, they are evaluated on access and not on define.
 
 The built-in field attributes are:
 
-- `group`: returned group name, e.g., "A Title?"
+- `key_obj`: model returned object, e.g., "A Title?"
-- `key`: slugified group value, e.g., "a-title"
+- `key`: slugified value of `key_obj`, e.g., "a-title"
-- `slug`: url path after root node, e.g. "config/a-title.html" (can be `None`)
 - `record`: parent node, e.g., `Page(path="/")`
-- `children`: dictionary of `{record: extras}` pairs
+- `slug`: url path under parent node, e.g. "config/a-title.html" (can be `None`)
-- `first_child`: first page
+- `children`: the elements of the grouping (a `Query` of `Record` type)
-- `first_extra`: first extra
 - `config`: configuration object (see below)
 
-Without any changes, the `key` value will just be `slugify(group)`.
-However, the other mapping `.key_map` will replace `group` with whatever replacement value is provided in the `.key_map` and then slugified.
-You could, for example, add a `C# = c-sharp` mapping, which would otherwise just be slugified to `c`.
-This is equivalent to `slugify(key_map.get(group))`.
-
 The `config` attribute contains the values that created the group:
 
 - `key`: attribute key, e.g., `TestA`
 - `root`: as provided by init, e.g., `/`
 - `slug`: the raw value, e.g., `config/{key}.html`
 - `template`: as provided by init, e.g., `example-config.html`
+- `key_obj_fn`: as provided by init, e.g., `X.upper() if X else 'empty'`
+- `replace_none_key`: as provided by init, e.g., `unknown`
 - `enabled`: boolean
 - `dependencies`: path to config file (if initialized from config)
 - `fields`: raw values from `TestA.fields`
 - `key_map`: raw values from `TestA.key_map`
+- `pagination`: raw values from `TestA.pagination`
+- `order_by`: list of key-strings from `TestA.children.order_by`
 
-In your template file you have access to the attributes, config, and children (pages):
 
-```jinja2
+### Config Subsection: .key_map
+
+Without any changes, the `key` value will just be `slugify(key_obj)`.
+However, the `.key_map` subsection will replace `key_obj` with whatever replacement value is provided in the mapping and is then slugified.
+Take the given example, `C# = c-sharp`, which would otherwise be slugified to `c`.
+This is equivalent to `slugify(key_map.get(key_obj))`.
+
+
+### Config Template
+
+In your template file ([`templates/example-config.html`](./templates/example-config.html)), you have access to the aforementioned attributes:
+
+
+```jinja
 <h2>{{ this.title }}</h2>
 <p>Key: {{ this.key }}, Attribute: {{ this.config.key }}</p>
 <ul>
@@ -155,14 +193,21 @@ def on_groupby_before_build_all(self, groupby, builder, **extra):
         # Yield groups
         value = args.field  # type: list # since model is 'strings' type
         for tag in value:
-            yield tag, {'tags_in_page': value}
+            yield tag
 ```
 
 This example is roughly equivalent to the config example above – the parameters of the `groupby.add_watcher` function correspond to the same config parameters.
 Additionally, you can set other types in `set_fields` (all strings are evaluated in jinja context!).
+Refer to `lektor_simple.py` for all available configuration options.
 
-`@watcher.grouping` sets the callback to generate group keys.
+The `@watcher.grouping` callback generates all groups for a single watcher-attribute.
-It has one optional flatten parameter:
+The callback body **can** produce groupings but does not have to.
+If you choose to produce an entry, you have to `yield` a grouping object (string, int, bool, float, or object).
+In any case, `key_obj` is slugified (see above) and then used to combine & cluster pages.
+You can yield more than one entry per source.
+Or ignore pages if you don't yield anything.
+
+The `@watcher.grouping` decorator takes one optional parameter:
 
 - `flatten` determines how Flow elements are processed.
   If `False`, the callback function is called once per Flow element.
@@ -182,21 +227,13 @@ k = args.key
 field = args.record[k.fieldKey].blocks[k.flowIndex].get(k.flowKey)
 ```
 
-The callback body **can** produce groupings but does not have to.
+Again, you can use all properties in your template.
-If you choose to produce an entry, you have to `yield` a string or tuple pair `(group, extra-info)`.
-`group` is slugified (see above) and then used to combine & cluster pages.
-The `extra-info` (optional) is passed through to your template file.
-You can yield more than one entry per source.
-Or ignore pages if you don't yield anything.
 
-The template file can access and display the `extra-info`:
+```jinja
-
-```jinja2
 <p>Custom field date: {{this.date}}</p>
 <ul>
-{%- for child, extras in this.children.items() -%}
+{%- for child in this.children %}
-{%- set etxra = (extras|first).tags_in_page %}
+<li>page "{{child.path}}" with tags: {{child.tags}}</li>
-<li>{{etxra|length}} tags on page "{{child.path}}": {{etxra}}</li>
 {%- endfor %}
 </ul>
 ```
@@ -227,29 +264,43 @@ def on_groupby_before_build_all(self, groupby, builder, **extra):
         return
 
     # load config directly (which also tracks dependency)
-    watcher = groupby.add_watcher('testC', config)
+    watcher = groupby.add_watcher('testC', config, pre_build=True)
 
     @watcher.grouping()
     def convert_replace_example(args):
         # args.field assumed to be Markdown
         obj = args.field.source
-        slugify_map = {}  # type Dict[str, str]
+        url_map = {}  # type Dict[str, str]
         for match in regex.finditer(obj):
             tag = match.group(1)
-            key = yield tag
+            vobj = yield tag
-            print('[advanced] slugify:', tag, '->', key)
+            if not hasattr(vobj, 'custom_attr'):
-            slugify_map[tag] = key
+                vobj.custom_attr = []
+            vobj.custom_attr.append(tag)
+            url_map[tag] = vobj.url_path
+            print('[advanced] slugify:', tag, '->', vobj.key)
 
         def _fn(match: re.Match) -> str:
             tag = match.group(1)
-            return f'<a href="/advanced/{slugify_map[tag]}/">{tag}</a>'
+            return f'<a href="{url_map[tag]}">{tag}</a>'
         args.field.source = regex.sub(_fn, obj)
 ```
 
 Notice, `add_watcher` accepts a config file as parameter which keeps also track of dependencies and rebuilds pages when you edit the config file.
-Further, the `yield` call returns the slugified group-key.
+Further, the `yield` call returns a `GroupBySource` virtual object.
-First, you do not need to slugify it yourself and second, potential replacements from `key_map` are already handled.
+You can use this object to add custom static attributes (similar to dynamic attributes with the `.fields` subsection config).
 
+Not all attributes are available at this time, as the grouping is still in progress.
+But you can use `vobj.url_path` to get the target URL or `vobj.key` to get the slugified object-key (substitutions from `key_map` are already applied).
+
+Usually, the grouping is postponed until the very end of the build process.
+However, in this case we want to modify the source before it is build by Lektor.
+For this situation we need to set `pre_build=True` in our `groupby.add_watcher()` call.
+All watcher with this flag will be processed before any Page is built.
+**Note:** If you can, avoid this performance regression.
+The grouping for these watchers will be performed each time you navigate from one page to another.
+
+This example uses a Markdown model type as source.
 For Markdown fields, we can modify the `source` attribute directly.
 All other field types need to be accessed via `args.record` key indirection (see [simple example](#simple-example)).
 
@@ -263,27 +314,50 @@ template = example-advanced.html
 match = {{([^}]{1,32})}}
 ```
 
-The config file takes the same parameters as the [config example](#quick-config).
+The config file takes the same parameters as the [config example](#config-file).
-As you can see, `slug` is evaluated in jinja context.
+We introduce a new config option `testC.pattern.match`.
-
-We introduced a new config option `testC.pattern.match`.
 This regular expression matches `{{` + any string less than 32 characters + `}}`.
-Notice, the parenthesis (`()`) will match only the inner part but the replace function (`re.sub`) will remove the `{{}}`.
+Notice, the parenthesis (`()`) will match only the inner part, thus the replace function (`re.sub`) removes the `{{}}`.
 
 
 
 ## Misc
 
-It was shortly mentioned above that slugs can be `None` (only if manually set to `slug = None`).
+### Omit output with empty slugs
-This is useful if you do not want to create subpages but rather an index page containing all groups.
-This can be done in combination with the next use-case:
 
-```jinja2
+It was shortly mentioned above that slugs can be `None` (e.g., manually set to `slug = None`).
-{%- for x in this|vgroups('TestA', 'TestB', recursive=True)|unique|sort %} 
+This is useful if you do not want to create subpages but rather an index page containing all groups.
-<a href="{{ x|url }}">({{ x.group }})</a>
+You can combine this with the next use-case.
+
+
+### Index pages & Group query + filter
+
+```jinja
+{%- for x in this|vgroups(keys=['TestA', 'TestB'], fields=[], flows=[], recursive=True, order_by='key_obj') %} 
+<a href="{{ x|url }}">({{ x.key_obj }})</a>
 {%- endfor %} 
 ```
 
 You can query the groups of any parent node (including those without slug).
+[`templates/page.html`](./templates/page.html) uses this.
 The keys (`'TestA', 'TestB'`) can be omitted which will return all groups of all attributes (you can still filter them with `x.config.key == 'TestC'`).
-Refer to [`templates/page.html`](./templates/page.html) for usage.
+The `fields` and `flows` params are also optional.
+With these you can match groups in `args.key.fieldKey` and `args.key.flowKey`.
+For example, if you have a “main-tags” field and an “additional-tags” field and you want to show the main-tags in a preview but both tags on a detail page.
+
+
+### Sorting groups
+
+Sorting is supported for the `vgroups` filter as well as for the children of each group (via config subsection `.children.order_by`).
+Coming back to the previous example, `order_by` can be either a comma-separated string of keys or a list of strings.
+Again, same [order-by](https://www.getlektor.com/docs/guides/page-order/) rules apply as for any other Lektor `Record`.
+Only this time, the attributes of the `GroupBy` object are used for sorting (including those you defined in the `.fields` subsection).
+
+
+### Pagination
+
+You may use the `.pagination` subsection or `watcher.config.set_pagination()` to configure a pagination controller.
+The `url_path` of a paginated Page depends on your `slug` value.
+If the slug ends on `/` or `/index.html`, Lektor will append `page/2/index.html` to the second page.
+If the slug contains a `.` (e.g. `/a/{key}.html`), Lektor will insert `page2` in front of the extension (e.g., `/a/{key}page2.html`).
+If you supply a different `url_suffix`, for example “.X.”, those same two urls will become `.X./2/index.html` and `/a/{key}.X.2.html` respectively.

examples/configs/advanced.ini

@@ -7,7 +7,7 @@ template = example-advanced.html
 match = {{([^}]{1,32})}}
 
 [testC.fields]
-desc = "Tag: " ~ this.group ~ ", Key: " ~ this.key
+desc = "Input object: {}, output key: {}".format(this.key_obj, this.key)
 
 [testC.key_map]
 Blog = case-sensitive

examples/configs/groupby.ini

@@ -4,9 +4,19 @@ root = /
 slug = config/{key}.html
 template = example-config.html
 split = ' '
+key_obj_fn = '{}-z-{}'.format(X.upper(), ARGS.key.fieldKey) if X else None
+replace_none_key = unknown
+
+[testA.children]
+order_by = -title, body
+
+[testA.pagination]
+enabled = true
+per_page = 1
+url_suffix = .page.
 
 [testA.fields]
-title = "Tagged: " ~ this.group
+title = "Tagged: " ~ this.key_obj
 
 [testA.key_map]
 Blog = News

examples/packages/advanced-example/lektor_advanced.py

@@ -17,20 +17,24 @@ class AdvancedGroupByPlugin(Plugin):
             return
 
         # load config directly (which also tracks dependency)
-        watcher = groupby.add_watcher('testC', config)
+        watcher = groupby.add_watcher('testC', config, pre_build=True)
 
         @watcher.grouping()
         def _replace(args: GroupByCallbackArgs) -> Generator[str, str, None]:
             # args.field assumed to be Markdown
             obj = args.field.source
-            slugify_map = {}  # type Dict[str, str]
+            url_map = {}  # type Dict[str, str]
             for match in regex.finditer(obj):
                 tag = match.group(1)
-                key = yield tag
+                vobj = yield tag
-                print('[advanced] slugify:', tag, '->', key)
+                if not hasattr(vobj, 'custom_attr'):
-                slugify_map[tag] = key
+                    vobj.custom_attr = []
+                # update static custom attribute
+                vobj.custom_attr.append(tag)
+                url_map[tag] = vobj.url_path
+                print('[advanced] slugify:', tag, '->', vobj.key)
 
             def _fn(match: re.Match) -> str:
                 tag = match.group(1)
-                return f'<a href="/advanced/{slugify_map[tag]}/">{tag}</a>'
+                return f'<a href="{url_map[tag]}">{tag}</a>'
             args.field.source = regex.sub(_fn, obj)

examples/packages/simple-example/lektor_simple.py

@@ -11,16 +11,24 @@ class SimpleGroupByPlugin(Plugin):
             'root': '/blog',
             'slug': 'simple/{key}/index.html',
             'template': 'example-simple.html',
+            'key_obj_fn': 'X.upper() if X else "empty"',
+            'replace_none_key': 'unknown',
         })
         watcher.config.set_key_map({'Foo': 'bar'})
         watcher.config.set_fields({'date': datetime.now()})
+        watcher.config.set_order_by('-title,body')
+        watcher.config.set_pagination(
+            enabled=True,
+            per_page=1,
+            url_suffix='p',
+        )
 
         @watcher.grouping(flatten=True)
         def fn_simple(args: GroupByCallbackArgs) -> Iterator[Tuple[str, dict]]:
             # Yield groups
             value = args.field  # type: list # since model is 'strings' type
             for tag in value:
-                yield tag, {'tags_in_page': value}
+                yield tag
             # Everything below is just for documentation purposes
             page = args.record  # extract additional info from source
             fieldKey, flowIndex, flowKey = args.key  # or get field index

examples/templates/example-advanced.html

@@ -1,4 +1,5 @@
 <h2>Path: {{ this | url(absolute=True) }}</h2>
 <p>This is: {{this}}</p>
 <p>Custom field, desc: "{{this.desc}}"</p>
-<p>Children: {{this.children}}</p>
+<p>Custom static, seen objects: {{this.custom_attr}}</p>
+<p>Children: {{this.children.all()}}</p>

examples/templates/example-config.html

@@ -1,7 +1,7 @@
 <h2>Path: {{ this | url(absolute=True) }}</h2>
 <p>This is: {{this}}</p>
-<p>Group: "{{this.group}}", Key: "{{this.key}}"</p>
+<p>Object: "{{this.key_obj}}", Key: "{{this.key}}"</p>
-<p>Custom field title: {{this.title}}</p>
+<p>Custom field title: "{{this.title}}"</p>
 <ul>
 {%- for child in this.children %}
 <li>Child: <a href="{{child|url}}">{{child.title}}</a> ({{child.path}})</li>

examples/templates/example-simple.html

@@ -1,9 +1,10 @@
 <h2>Path: {{ this | url(absolute=True) }}</h2>
 <p>This is: {{this}}</p>
+<p>Key: {{this.key}}</p>
+<p>Object: {{this.key_obj}}</p>
 <p>Custom field date: {{this.date}}</p>
 <ul>
-{%- for child, extras in this.children.items() -%}
+{%- for child in this.children %}
-{%- set etxra = (extras|first).tags_in_page %}
+<li>page "{{child.path}}" with tags: {{child.tags}}</li>
-<li>{{etxra|length}} tags on page "{{child.path}}": {{etxra}}</li>
 {%- endfor %}
 </ul>

examples/templates/page.html

@@ -20,7 +20,7 @@ main { margin: 3em; }
   <footer>
     {%- for k, v in [('testA','Config'),('testB','Simple'),('testC','Advanced')] %}
     <div>{{v}} Tags:
-      {%- for x in this|vgroups(k, recursive=True)|unique|sort %} 
+      {%- for x in this|vgroups(k, recursive=True, order_by='key_obj') %} 
         <a href="{{ x|url }}">({{x.key}})</a>
       {%- endfor %} 
     </div>

lektor_groupby/backref.py

@@ -1,13 +1,21 @@
 from lektor.context import get_ctx
-from typing import TYPE_CHECKING, Iterator
+from typing import TYPE_CHECKING, Set, List, Dict, Union, Iterable, Iterator
-from weakref import WeakSet
+import weakref
+from .util import split_strip
 if TYPE_CHECKING:
     from lektor.builder import Builder
     from lektor.db import Record
     from .groupby import GroupBy
+    from .model import FieldKeyPath
     from .vobj import GroupBySource
 
 
+class WeakVGroupsList(list):
+    def add(self, strong: 'FieldKeyPath', weak: 'GroupBySource') -> None:
+        super().append((strong, weakref.ref(weak)))
+        # super().append((strong, weak))  # strong-ref
+
+
 class GroupByRef:
     @staticmethod
     def of(builder: 'Builder') -> 'GroupBy':
@@ -22,7 +30,7 @@ class GroupByRef:
 
 class VGroups:
     @staticmethod
-    def of(record: 'Record') -> WeakSet:
+    def of(record: 'Record') -> WeakVGroupsList:
         '''
         Return the (weak) set of virtual objects of a page.
         Creates a new set if it does not exist yet.
@@ -30,30 +38,77 @@ class VGroups:
         try:
             wset = record.__vgroups  # type: ignore[attr-defined]
         except AttributeError:
-            wset = WeakSet()
+            wset = WeakVGroupsList()
             record.__vgroups = wset  # type: ignore[attr-defined]
         return wset  # type: ignore[no-any-return]
 
     @staticmethod
-    def iter(record: 'Record', *keys: str, recursive: bool = False) \
+    def iter(
-            -> Iterator['GroupBySource']:
+        record: 'Record',
+        keys: Union[str, Iterable[str], None] = None,
+        *,
+        fields: Union[str, Iterable[str], None] = None,
+        flows: Union[str, Iterable[str], None] = None,
+        order_by: Union[str, Iterable[str], None] = None,
+        recursive: bool = True,
+        unique: bool = True,
+    ) -> Iterator['GroupBySource']:
         ''' Extract all referencing groupby virtual objects from a page. '''
+        # prepare filter
+        if isinstance(keys, str):
+            keys = [keys]
+        if isinstance(fields, str):
+            fields = [fields]
+        if isinstance(flows, str):
+            flows = [flows]
+        # get GroupBy object
         ctx = get_ctx()
         if not ctx:
             raise NotImplementedError("Shouldn't happen, where is my context?")
-        # get GroupBy object
         builder = ctx.build_state.builder
-        groupby = GroupByRef.of(builder)
+        GroupByRef.of(builder).make_once(keys)  # ensure did cluster before use
-        groupby.make_once(builder)  # ensure did cluster before
-        # manage config dependencies
-        for dep in groupby.dependencies:
-            ctx.record_dependency(dep)
         # find groups
         proc_list = [record]
+
+        # Note: An ordered Set would be more approptiate but there is none.
+        # So lets use the insert order of dict (guaranteed since Python 3.7)
+        _only_uniques = {}  # type: Dict[GroupBySource, None]
+        _w_duplicates = []  # type: List[GroupBySource]
+
         while proc_list:
             page = proc_list.pop(0)
             if recursive and hasattr(page, 'children'):
-                proc_list.extend(page.children)  # type: ignore[attr-defined]
+                proc_list.extend(page.children)
-            for vobj in VGroups.of(page):
+            for key, vobj in VGroups.of(page):
-                if not keys or vobj.config.key in keys:
+                if fields and key.fieldKey not in fields:
-                    yield vobj
+                    continue
+                if flows and key.flowKey not in flows:
+                    continue
+                if keys and vobj().config.key not in keys:
+                    continue
+                if unique:
+                    _only_uniques[vobj()] = None  # we only need the keys()
+                else:
+                    _w_duplicates.append(vobj())
+
+        done_list = _only_uniques if unique else _w_duplicates
+
+        # manage config dependencies
+        deps = set()  # type: Set[str]
+        for vobj in done_list:
+            deps.update(vobj.config.dependencies)
+            # ctx.record_virtual_dependency(vobj) # TODO: needed? works without
+        for dep in deps:
+            ctx.record_dependency(dep)
+
+        if order_by:
+            if isinstance(order_by, str):
+                order = split_strip(order_by, ',')  # type: Iterable[str]
+            elif isinstance(order_by, (list, tuple)):
+                order = order_by
+            else:
+                raise AttributeError('order_by must be str or list type.')
+            # using get_sort_key() of GroupBySource
+            yield from sorted(done_list, key=lambda x: x.get_sort_key(order))
+        else:
+            yield from done_list

lektor_groupby/config.py

@@ -1,11 +1,34 @@
 from inifile import IniFile
-from lektor.utils import slugify
+from lektor.environment import Expression
+from lektor.context import Context
+from lektor.utils import slugify as _slugify
+from typing import (
+    TYPE_CHECKING, Set, Dict, Optional, Union, Any, List, Generator
+)
+from .util import split_strip
+if TYPE_CHECKING:
+    from lektor.sourceobj import SourceObject
 
-from typing import Set, Dict, Optional, Union, Any
 
 AnyConfig = Union['Config', IniFile, Dict]
 
 
+class ConfigError(Exception):
+    ''' Used to print a Lektor console error. '''
+
+    def __init__(
+        self, key: str, field: str, expr: str, error: Union[Exception, str]
+    ):
+        self.key = key
+        self.field = field
+        self.expr = expr
+        self.error = error
+
+    def __str__(self) -> str:
+        return 'Invalid config for [{}.{}] = "{}"  –  Error: {}'.format(
+            self.key, self.field, self.expr, repr(self.error))
+
+
 class Config:
     '''
     Holds information for GroupByWatcher and GroupBySource.
@@ -21,20 +44,27 @@ class Config:
         root: Optional[str] = None,  # default: "/"
         slug: Optional[str] = None,  # default: "{attr}/{group}/index.html"
         template: Optional[str] = None,  # default: "groupby-{attr}.html"
+        replace_none_key: Optional[str] = None,  # default: None
+        key_obj_fn: Optional[str] = None,  # default: None
     ) -> None:
         self.key = key
         self.root = (root or '/').rstrip('/') or '/'
         self.slug = slug or (key + '/{key}/')  # key = GroupBySource.key
         self.template = template or f'groupby-{self.key}.html'
+        self.replace_none_key = replace_none_key
+        self.key_obj_fn = key_obj_fn
         # editable after init
         self.enabled = True
         self.dependencies = set()  # type: Set[str]
         self.fields = {}  # type: Dict[str, Any]
         self.key_map = {}  # type: Dict[str, str]
+        self.pagination = {}  # type: Dict[str, Any]
+        self.order_by = None  # type: Optional[List[str]]
 
     def slugify(self, k: str) -> str:
         ''' key_map replace and slugify. '''
-        return slugify(self.key_map.get(k, k))  # type: ignore[no-any-return]
+        rv = self.key_map.get(k, k)
+        return _slugify(rv) or rv  # the `or` allows for example "_"
 
     def set_fields(self, fields: Optional[Dict[str, Any]]) -> None:
         '''
@@ -48,11 +78,32 @@ class Config:
         ''' This mapping replaces group keys before slugify. '''
         self.key_map = key_map or {}
 
+    def set_pagination(
+        self,
+        enabled: Optional[bool] = None,
+        per_page: Optional[int] = None,
+        url_suffix: Optional[str] = None,
+        items: Optional[str] = None,
+    ) -> None:
+        ''' Used for pagination. '''
+        self.pagination = dict(
+            enabled=enabled,
+            per_page=per_page,
+            url_suffix=url_suffix,
+            items=items,
+        )
+
+    def set_order_by(self, order_by: Optional[str]) -> None:
+        ''' If specified, children will be sorted according to keys. '''
+        self.order_by = split_strip(order_by or '', ',') or None
+
     def __repr__(self) -> str:
         txt = '<GroupByConfig'
-        for x in ['key', 'root', 'slug', 'template', 'enabled']:
+        for x in ['enabled', 'key', 'root', 'slug', 'template', 'key_obj_fn']:
             txt += ' {}="{}"'.format(x, getattr(self, x))
         txt += f' fields="{", ".join(self.fields)}"'
+        if self.order_by:
+            txt += ' order_by="{}"'.format(' ,'.join(self.order_by))
         return txt + '>'
 
     @staticmethod
@@ -63,6 +114,8 @@ class Config:
             root=cfg.get('root'),
             slug=cfg.get('slug'),
             template=cfg.get('template'),
+            replace_none_key=cfg.get('replace_none_key'),
+            key_obj_fn=cfg.get('key_obj_fn'),
         )
 
     @staticmethod
@@ -74,6 +127,13 @@ class Config:
         conf.dependencies.add(ini.filename)
         conf.set_fields(ini.section_as_dict(key + '.fields'))
         conf.set_key_map(ini.section_as_dict(key + '.key_map'))
+        conf.set_pagination(
+            enabled=ini.get_bool(key + '.pagination.enabled', None),
+            per_page=ini.get_int(key + '.pagination.per_page', None),
+            url_suffix=ini.get(key + '.pagination.url_suffix'),
+            items=ini.get(key + '.pagination.items'),
+        )
+        conf.set_order_by(ini.get(key + '.children.order_by', None))
         return conf
 
     @staticmethod
@@ -85,3 +145,56 @@ class Config:
             return Config.from_ini(key, config)
         elif isinstance(config, Dict):
             return Config.from_dict(key, config)
+
+    # -----------------------------------
+    #          Field Expressions
+    # -----------------------------------
+
+    def _make_expression(self, expr: Any, *, on: 'SourceObject', field: str) \
+            -> Union[Expression, Any]:
+        ''' Create Expression and report any config error. '''
+        if not isinstance(expr, str):
+            return expr
+        try:
+            return Expression(on.pad.env, expr)
+        except Exception as e:
+            raise ConfigError(self.key, field, expr, e)
+
+    def eval_field(self, attr: str, *, on: 'SourceObject') \
+            -> Union[Expression, Any]:
+        ''' Create an expression for a custom defined user field. '''
+        # do not `gather_dependencies` because fields are evaluated on the fly
+        # dependency tracking happens whenever a field is accessed
+        return self._make_expression(
+            self.fields[attr], on=on, field='fields.' + attr)
+
+    def eval_slug(self, key: str, *, on: 'SourceObject') -> Optional[str]:
+        ''' Either perform a "{key}" substitution or evaluate expression. '''
+        cfg_slug = self.slug
+        if not cfg_slug:
+            return None
+        if '{key}' in cfg_slug:
+            if key:
+                return cfg_slug.replace('{key}', key)
+            else:
+                raise ConfigError(self.key, 'slug', cfg_slug,
+                                  'Cannot replace {key} with None')
+                return None
+        else:
+            # TODO: do we need `gather_dependencies` here too?
+            expr = self._make_expression(cfg_slug, on=on, field='slug')
+            return expr.evaluate(on.pad, this=on, alt=on.alt) or None
+
+    def eval_key_obj_fn(self, *, on: 'SourceObject', context: Dict) -> Any:
+        '''
+        If `key_obj_fn` is set, evaluate field expression.
+        Note: The function does not check whether `key_obj_fn` is set.
+        Return: A Generator result is automatically unpacked into a list.
+        '''
+        exp = self._make_expression(self.key_obj_fn, on=on, field='key_obj_fn')
+        with Context(pad=on.pad) as ctx:
+            with ctx.gather_dependencies(self.dependencies.add):
+                res = exp.evaluate(on.pad, this=on, alt=on.alt, values=context)
+        if isinstance(res, Generator):
+            res = list(res)  # unpack for 1-to-n replacement
+        return res

lektor_groupby/groupby.py

@@ -1,12 +1,13 @@
 from lektor.builder import PathCache
 from lektor.db import Record  # isinstance
-from typing import TYPE_CHECKING, Set, List
+from lektor.reporter import reporter  # build
+from typing import TYPE_CHECKING, List, Optional, Iterable
 from .config import Config
 from .watcher import Watcher
 if TYPE_CHECKING:
-    from .config import AnyConfig
     from lektor.builder import Builder
     from lektor.sourceobj import SourceObject
+    from .config import AnyConfig
     from .resolver import Resolver
     from .vobj import GroupBySource
 
@@ -19,62 +20,99 @@ class GroupBy:
     '''
 
     def __init__(self, resolver: 'Resolver') -> None:
+        self._building = False
         self._watcher = []  # type: List[Watcher]
         self._results = []  # type: List[GroupBySource]
+        self._pre_build_priority = []  # type: List[str]  # config.key
         self.resolver = resolver
 
-    def add_watcher(self, key: str, config: 'AnyConfig') -> Watcher:
+    @property
+    def isBuilding(self) -> bool:
+        return self._building
+
+    def add_watcher(
+        self, key: str, config: 'AnyConfig', *, pre_build: bool = False
+    ) -> Watcher:
         ''' Init Config and add to watch list. '''
         w = Watcher(Config.from_any(key, config))
         self._watcher.append(w)
+        if pre_build:
+            self._pre_build_priority.append(w.config.key)
         return w
 
-    def get_dependencies(self) -> Set[str]:
-        deps = set()  # type: Set[str]
-        for w in self._watcher:
-            deps.update(w.config.dependencies)
-        return deps
-
     def queue_all(self, builder: 'Builder') -> None:
         ''' Iterate full site-tree and queue all children. '''
-        self.dependencies = self.get_dependencies()
         # remove disabled watchers
         self._watcher = [w for w in self._watcher if w.config.enabled]
         if not self._watcher:
             return
         # initialize remaining (enabled) watchers
         for w in self._watcher:
-            w.initialize(builder.pad.db)
+            w.initialize(builder.pad)
         # iterate over whole build tree
         queue = builder.pad.get_all_roots()  # type: List[SourceObject]
         while queue:
             record = queue.pop()
             if hasattr(record, 'attachments'):
-                queue.extend(record.attachments)  # type: ignore[attr-defined]
+                queue.extend(record.attachments)
             if hasattr(record, 'children'):
-                queue.extend(record.children)  # type: ignore[attr-defined]
+                queue.extend(record.children)
             if isinstance(record, Record):
                 for w in self._watcher:
                     if w.should_process(record):
-                        w.process(record)
+                        w.remember(record)
+        # build sources which need building before actual lektor build
+        if self._pre_build_priority:
+            self.make_once(self._pre_build_priority)
+            self._pre_build_priority.clear()
 
-    def make_once(self, builder: 'Builder') -> None:
+    def make_once(self, filter_keys: Optional[Iterable[str]] = None) -> None:
-        ''' Perform groupby, iter over sources with watcher callback. '''
+        '''
-        if self._watcher:
+        Perform groupby, iter over sources with watcher callback.
-            self.resolver.reset()
+        If `filter_keys` is set, ignore all other watchers.
-            for w in self._watcher:
+        '''
-                root = builder.pad.get(w.config.root)
+        if not self._watcher:
-                for vobj in w.iter_sources(root):
+            return
-                    self._results.append(vobj)
+        remaining = []
-                    self.resolver.add(vobj)
+        for w in self._watcher:
-            self._watcher.clear()
+            # only process vobjs that are used somewhere
+            if filter_keys and w.config.key not in filter_keys:
+                remaining.append(w)
+                continue
+            self.resolver.reset(w.config.key)
+            # these are used in the current context (or on `build_all`)
+            for vobj in w.iter_sources():
+                # add original source
+                self._results.append(vobj)
+                self.resolver.add(vobj)
+                # and also add pagination sources
+                for sub_vobj in vobj.__iter_pagination_sources__():
+                    self._results.append(sub_vobj)
+                    self.resolver.add(sub_vobj)
+        # TODO: if this should ever run concurrently, pop() from watchers
+        self._watcher = remaining
 
-    def build_all(self, builder: 'Builder') -> None:
+    def build_all(
-        ''' Create virtual objects and build sources. '''
+        self,
-        self.make_once(builder)  # in case no page used the |vgroups filter
+        builder: 'Builder',
-        path_cache = PathCache(builder.env)
+        specific: Optional['GroupBySource'] = None
-        for vobj in self._results:
+    ) -> None:
-            if vobj.slug:
+        '''
-                builder.build(vobj, path_cache)
+        Build actual artifacts (if needed).
-        del path_cache
+        If `specific` is set, only build the artifacts for that single vobj
-        self._results.clear()  # garbage collect weak refs
+        '''
+        if not self._watcher and not self._results:
+            return
+        with reporter.build('groupby', builder):  # type:ignore
+            # in case no page used the |vgroups filter
+            self.make_once([specific.config.key] if specific else None)
+            self._building = True
+            path_cache = PathCache(builder.env)
+            for vobj in self._results:
+                if specific and vobj.path != specific.path:
+                    continue
+                if vobj.slug:
+                    builder.build(vobj, path_cache)
+            del path_cache
+            self._building = False
+            self._results.clear()  # garbage collect weak refs

lektor_groupby/model.py

@@ -48,6 +48,7 @@ class ModelReader:
         for r_key, subs in self._models.get(record.datamodel.id, {}).items():
             field = record[r_key]
             if not field:
+                yield FieldKeyPath(r_key), field
                 continue
             if subs == '*':  # either normal field or flow type (all blocks)
                 if self.flatten and isinstance(field, Flow):

lektor_groupby/pagination.py

@@ -0,0 +1,29 @@
+from lektor import datamodel
+from typing import TYPE_CHECKING, Any, Dict
+if TYPE_CHECKING:
+    from lektor.environment import Environment
+    from lektor.pagination import Pagination
+    from lektor.sourceobj import SourceObject
+
+
+class PaginationConfig(datamodel.PaginationConfig):
+    # because original method does not work for virtual sources.
+    def __init__(self, env: 'Environment', config: Dict[str, Any], total: int):
+        super().__init__(env, **config)
+        self._total_items_count = total
+
+    @staticmethod
+    def get_record_for_page(record: 'SourceObject', page_num: int) -> Any:
+        for_page = getattr(record, '__for_page__', None)
+        if callable(for_page):
+            return for_page(page_num)
+        return datamodel.PaginationConfig.get_record_for_page(record, page_num)
+
+    def count_total_items(self, record: 'SourceObject') -> int:
+        ''' Override super() to prevent a record.children query. '''
+        return self._total_items_count
+
+    if TYPE_CHECKING:
+        def get_pagination_controller(self, record: 'SourceObject') \
+                -> 'Pagination':
+            ...

lektor_groupby/plugin.py

@@ -1,11 +1,12 @@
-from lektor.db import Page  # isinstance
+from lektor.assets import Asset  # isinstance
+from lektor.db import Record  # isinstance
 from lektor.pluginsystem import Plugin  # subclass
-from typing import TYPE_CHECKING, Iterator, Any
+from typing import TYPE_CHECKING, Set, Iterator, Any
 from .backref import GroupByRef, VGroups
 from .groupby import GroupBy
 from .pruner import prune
 from .resolver import Resolver
-from .vobj import VPATH, GroupBySource, GroupByBuildProgram
+from .vobj import GroupBySource, GroupByBuildProgram
 if TYPE_CHECKING:
     from lektor.builder import Builder, BuildState
     from lektor.sourceobj import SourceObject
@@ -17,33 +18,49 @@ class GroupByPlugin(Plugin):
     description = 'Cluster arbitrary records with field attribute keyword.'
 
     def on_setup_env(self, **extra: Any) -> None:
-        self.has_changes = False
         self.resolver = Resolver(self.env)
         self.env.add_build_program(GroupBySource, GroupByBuildProgram)
         self.env.jinja_env.filters.update(vgroups=VGroups.iter)
+        # kep track of already rebuilt GroupBySource artifacts
+        self._is_build_all = False
+        self._has_been_built = set()  # type: Set[str]
+
+    def on_before_build_all(self, **extra: Any) -> None:
+        self._is_build_all = True
 
     def on_before_build(
         self, builder: 'Builder', source: 'SourceObject', **extra: Any
     ) -> None:
         # before-build may be called before before-build-all (issue #1017)
-        # make sure it is always evaluated first
+        if isinstance(source, Asset):
-        if isinstance(source, Page):
+            return
-            self._init_once(builder)
+        # make GroupBySource available before building any Record artifact
+        groupby = self._init_once(builder)
+        # special handling for self-building of GroupBySource artifacts
+        if isinstance(source, GroupBySource):
+            if groupby.isBuilding:  # build is during groupby.build_all()
+                self._has_been_built.add(source.path)
+            elif source.path not in self._has_been_built:
+                groupby.build_all(builder, source)  # needs rebuilding
 
-    def on_after_build(self, build_state: 'BuildState', **extra: Any) -> None:
+    def on_after_build(
-        if build_state.updated_artifacts:
+        self, source: 'SourceObject', build_state: 'BuildState', **extra: Any
-            self.has_changes = True
+    ) -> None:
+        # a normal page update. We may need to re-build our GroupBySource
+        if not self._is_build_all and isinstance(source, Record):
+            if build_state.updated_artifacts:
+                # TODO: instead of clear(), only remove affected GroupBySource
+                #       ideally, identify which file has triggered the re-build
+                self._has_been_built.clear()
 
     def on_after_build_all(self, builder: 'Builder', **extra: Any) -> None:
-        # only rebuild if has changes (bypass idle builds)
+        # by now, most likely already built. So, build_all() is a no-op
-        # or the very first time after startup (url resolver & pruning)
+        self._init_once(builder).build_all(builder)
-        if self.has_changes or not self.resolver.has_any:
+        self._is_build_all = False
-            self._init_once(builder).build_all(builder)  # updates resolver
-            self.has_changes = False
 
     def on_after_prune(self, builder: 'Builder', **extra: Any) -> None:
         # TODO: find a better way to prune unreferenced elements
-        prune(builder, VPATH, self.resolver.files)
+        prune(builder, self.resolver.files)
 
     # ------------
     #   internal
@@ -75,7 +92,11 @@ class GroupByPlugin(Plugin):
             @watcher.grouping()
             def _fn(args: 'GroupByCallbackArgs') -> Iterator[str]:
                 val = args.field
-                if isinstance(val, str):
+                if isinstance(val, str) and val != '':
                     val = map(str.strip, val.split(split)) if split else [val]
+                elif isinstance(val, (bool, int, float)):
+                    val = [val]
+                elif not val:  # after checking for '', False, 0, and 0.0
+                    val = [None]
                 if isinstance(val, (list, map)):
                     yield from val

lektor_groupby/pruner.py

@@ -1,45 +1,77 @@
 '''
-Static collector for build-artifact urls.
+Usage:
-All non-tracked VPATH-urls will be pruned after build.
+  VirtualSourceObject.produce_artifacts()
+    -> remember url and later supply as `current_urls`
+  VirtualSourceObject.build_artifact()
+    -> `get_ctx().record_virtual_dependency(VirtualPruner())`
 '''
 from lektor.reporter import reporter  # report_pruned_artifact
+from lektor.sourceobj import VirtualSourceObject  # subclass
 from lektor.utils import prune_file_and_folder
-from typing import TYPE_CHECKING, Set, Iterable
+import os
+from typing import TYPE_CHECKING, Set, List, Iterable
 if TYPE_CHECKING:
     from lektor.builder import Builder
+    from sqlite3 import Connection
 
 
-def _normalize_url_cache(url_cache: Iterable[str]) -> Set[str]:
+class VirtualPruner(VirtualSourceObject):
+    ''' Indicate that a generated VirtualSourceObject has pruning support. '''
+    VPATH = '/@VirtualPruner'
+
+    def __init__(self) -> None:
+        self._path = VirtualPruner.VPATH  # if needed, add suffix variable
+
+    @property
+    def path(self) -> str:  # type: ignore[override]
+        return self._path
+
+
+def prune(builder: 'Builder', current_urls: Iterable[str]) -> None:
+    ''' Removes previously generated, but now unreferenced Artifacts. '''
+    dest_dir = builder.destination_path
+    con = builder.connect_to_database()
+    try:
+        previous = _query_prunable(con)
+        current = _normalize_urls(current_urls)
+        to_be_pruned = previous.difference(current)
+        for file in to_be_pruned:
+            reporter.report_pruned_artifact(file)  # type: ignore
+            prune_file_and_folder(os.path.join(
+                dest_dir, file.strip('/').replace('/', os.path.sep)), dest_dir)
+        # if no exception raised, update db to remove obsolete references
+        _prune_db_artifacts(con, list(to_be_pruned))
+    finally:
+        con.close()
+
+
+# ---------------------------
+#   Internal helper methods
+# ---------------------------
+
+def _normalize_urls(urls: Iterable[str]) -> Set[str]:
     cache = set()
-    for url in url_cache:
+    for url in urls:
         if url.endswith('/'):
             url += 'index.html'
         cache.add(url.lstrip('/'))
     return cache
 
 
-def prune(builder: 'Builder', vpath: str, url_cache: Iterable[str]) -> None:
+def _query_prunable(conn: 'Connection') -> Set[str]:
-    '''
+    ''' Query database for artifacts that have the VirtualPruner dependency '''
-    Remove previously generated, unreferenced Artifacts.
+    cur = conn.cursor()
-    All urls in url_cache must have a trailing "/index.html" (instead of "/")
+    cur.execute('SELECT artifact FROM artifacts WHERE source = ?',
-    and also, no leading slash, "blog/index.html" instead of "/blog/index.html"
+                [VirtualPruner.VPATH])
-    '''
+    return set(x for x, in cur.fetchall())
-    vpath = '@' + vpath.lstrip('@')  # just in case of user error
+
-    dest_path = builder.destination_path
+
-    url_cache = _normalize_url_cache(url_cache)
+def _prune_db_artifacts(conn: 'Connection', urls: List[str]) -> None:
-    con = builder.connect_to_database()
+    ''' Remove obsolete artifact references from database. '''
-    try:
+    MAX_VARS = 999  # Default SQLITE_MAX_VARIABLE_NUMBER.
-        with builder.new_build_state() as build_state:
+    cur = conn.cursor()
-            for url, file in build_state.iter_artifacts():
+    for i in range(0, len(urls), MAX_VARS):
-                if url.lstrip('/') in url_cache:
+        batch = urls[i: i + MAX_VARS]
-                    continue  # generated in this build-run
+        cur.execute('DELETE FROM artifacts WHERE artifact in ({})'.format(
-                infos = build_state.get_artifact_dependency_infos(url, [])
+            ','.join(['?'] * len(batch))), batch)
-                for artifact_name, _ in infos:
+        conn.commit()
-                    if vpath not in artifact_name:
-                        continue  # we only care about our Virtuals
-                    reporter.report_pruned_artifact(url)
-                    prune_file_and_folder(file.filename, dest_path)
-                    build_state.remove_artifact(url)
-                    break  # there is only one VPATH-entry per source
-    finally:
-        con.close()

lektor_groupby/query.py

@@ -0,0 +1,80 @@
+# adapting https://github.com/dairiki/lektorlib/blob/master/lektorlib/query.py
+from lektor.constants import PRIMARY_ALT
+from lektor.db import Query  # subclass
+from typing import TYPE_CHECKING, List, Optional, Generator, Iterable
+if TYPE_CHECKING:
+    from lektor.db import Record, Pad
+
+
+class FixedRecordsQuery(Query):
+    def __init__(
+        self, pad: 'Pad', child_paths: Iterable[str], alt: str = PRIMARY_ALT
+    ):
+        ''' Query with a pre-defined list of children of type Record. '''
+        super().__init__('/', pad, alt=alt)
+        self.__child_paths = [x.lstrip('/') for x in child_paths]
+
+    def _get(
+        self, path: str, persist: bool = True, page_num: Optional[int] = None
+    ) -> Optional['Record']:
+        ''' Internal getter for a single Record. '''
+        if path not in self.__child_paths:
+            return None
+        if page_num is None:
+            page_num = self._page_num
+        return self.pad.get(  # type: ignore[no-any-return]
+            path, alt=self.alt, page_num=page_num, persist=persist)
+
+    def _iterate(self) -> Generator['Record', None, None]:
+        ''' Iterate over internal set of Record elements. '''
+        # ignore self record dependency from super()
+        for path in self.__child_paths:
+            record = self._get(path, persist=False)
+            if record is None:
+                if self._page_num is not None:
+                    # Sanity check: ensure the unpaginated version exists
+                    unpaginated = self._get(path, persist=False, page_num=None)
+                    if unpaginated is not None:
+                        # Requested explicit page_num, but source does not
+                        # support pagination.  Punt and skip it.
+                        continue
+                raise RuntimeError('could not load source for ' + path)
+
+            is_attachment = getattr(record, 'is_attachment', False)
+            if self._include_attachments and not is_attachment \
+                    or self._include_pages and is_attachment:
+                continue
+            if self._matches(record):
+                yield record
+
+    def get_order_by(self) -> Optional[List[str]]:
+        ''' Return list of attribute strings for sort order. '''
+        # ignore datamodel ordering from super()
+        return self._order_by  # type: ignore[no-any-return]
+
+    def count(self) -> int:
+        ''' Count matched objects. '''
+        if self._pristine:
+            return len(self.__child_paths)
+        return super().count()  # type: ignore[no-any-return]
+
+    @property
+    def total(self) -> int:
+        ''' Return total entries count (without any filter). '''
+        return len(self.__child_paths)
+
+    def get(self, path: str, page_num: Optional[int] = None) \
+            -> Optional['Record']:
+        ''' Return Record with given path '''
+        if path in self.__child_paths:
+            return self._get(path, page_num=page_num)
+        return None
+
+    def __bool__(self) -> bool:
+        if self._pristine:
+            return len(self.__child_paths) > 0
+        return super().__bool__()
+
+    if TYPE_CHECKING:
+        def request_page(self, page_num: Optional[int]) -> 'FixedRecordsQuery':
+            ...

lektor_groupby/resolver.py

@@ -1,6 +1,6 @@
-from lektor.db import Record  # isinstance
+from lektor.db import Page  # isinstance
-from lektor.utils import build_url
+from typing import TYPE_CHECKING, NamedTuple, Dict, List, Set, Any, Optional
-from typing import TYPE_CHECKING, Dict, List, Tuple, Optional, Iterable
+from .util import build_url
 from .vobj import VPATH, GroupBySource
 if TYPE_CHECKING:
     from lektor.environment import Environment
@@ -8,6 +8,21 @@ if TYPE_CHECKING:
     from .config import Config
 
 
+class ResolverEntry(NamedTuple):
+    key: str
+    key_obj: Any
+    config: 'Config'
+    page: Optional[int]
+
+    def equals(
+        self, path: str, conf_key: str, vobj_key: str, page: Optional[int]
+    ) -> bool:
+        return self.key == vobj_key \
+            and self.config.key == conf_key \
+            and self.config.root == path \
+            and self.page == page
+
+
 class Resolver:
     '''
     Resolve virtual paths and urls ending in /.
@@ -15,26 +30,34 @@ class Resolver:
     '''
 
     def __init__(self, env: 'Environment') -> None:
-        self._data = {}  # type: Dict[str, Tuple[str, Config]]
+        self._data = {}  # type: Dict[str, Dict[str, ResolverEntry]]
         env.urlresolver(self.resolve_server_path)
         env.virtualpathresolver(VPATH.lstrip('@'))(self.resolve_virtual_path)
 
     @property
     def has_any(self) -> bool:
-        return bool(self._data)
+        return any(bool(x) for x in self._data.values())
 
     @property
-    def files(self) -> Iterable[str]:
+    def files(self) -> Set[str]:
-        return self._data
+        return set(y for x in self._data.values() for y in x.keys())
 
-    def reset(self) -> None:
+    def reset(self, key: Optional[str] = None) -> None:
         ''' Clear previously recorded virtual objects. '''
-        self._data.clear()
+        if key:
+            if key in self._data:  # only delete if exists
+                del self._data[key]
+        else:
+            self._data.clear()
 
     def add(self, vobj: GroupBySource) -> None:
         ''' Track new virtual object (only if slug is set). '''
         if vobj.slug:
-            self._data[vobj.url_path] = (vobj.group, vobj.config)
+            # `page_num = 1` overwrites `page_num = None` -> same url_path()
+            if vobj.config.key not in self._data:
+                self._data[vobj.config.key] = {}
+            self._data[vobj.config.key][vobj.url_path] = ResolverEntry(
+                vobj.key, vobj.key_obj, vobj.config, vobj.page_num)
 
     # ------------
     #   Resolver
@@ -43,20 +66,30 @@ class Resolver:
     def resolve_server_path(self, node: 'SourceObject', pieces: List[str]) \
             -> Optional[GroupBySource]:
         ''' Local server only: resolve /tag/rss/ -> /tag/rss/index.html '''
-        if isinstance(node, Record):
+        if isinstance(node, Page):
-            rv = self._data.get(build_url([node.url_path] + pieces))
+            url = build_url([node.url_path] + pieces)
-            if rv:
+            for subset in self._data.values():
-                return GroupBySource(node, group=rv[0], config=rv[1])
+                rv = subset.get(url)
+                if rv:
+                    return GroupBySource(
+                        node, rv.key, rv.config, rv.page).finalize(rv.key_obj)
         return None
 
     def resolve_virtual_path(self, node: 'SourceObject', pieces: List[str]) \
             -> Optional[GroupBySource]:
         ''' Admin UI only: Prevent server error and null-redirect. '''
-        if isinstance(node, Record) and len(pieces) >= 2:
+        # format: /path/to/page@groupby/{config-key}/{vobj-key}/{page-num}
+        if isinstance(node, Page) and len(pieces) >= 2:
             path = node['_path']  # type: str
-            key, grp, *_ = pieces
+            conf_key, vobj_key, *optional_page = pieces
-            for group, conf in self._data.values():
+            page = None
-                if key == conf.key and path == conf.root:
+            if optional_page:
-                    if conf.slugify(group) == grp:
+                try:
-                        return GroupBySource(node, group, conf)
+                    page = int(optional_page[0])
+                except ValueError:
+                    pass
+            for rv in self._data.get(conf_key, {}).values():
+                if rv.equals(path, conf_key, vobj_key, page):
+                    return GroupBySource(
+                        node, rv.key, rv.config, rv.page).finalize(rv.key_obj)
         return None

lektor_groupby/util.py

@@ -1,24 +1,16 @@
-from lektor.reporter import reporter, style
+from typing import List, Dict, Optional, TypeVar
+from typing import Callable, Any, Union, Generic
 
-from typing import List, Dict
+T = TypeVar('T')
 
 
-def report_config_error(key: str, field: str, val: str, e: Exception) -> None:
+def most_used_key(keys: List[T]) -> Optional[T]:
-    ''' Send error message to Lektor reporter. Indicate which field is bad. '''
+    ''' Find string with most occurrences. '''
-    msg = '[ERROR] invalid config for [{}.{}] = "{}",  Error: {}'.format(
-        key, field, val, repr(e))
-    try:
-        reporter._write_line(style(msg, fg='red'))
-    except Exception:
-        print(msg)  # fallback in case Lektor API changes
-
-
-def most_used_key(keys: List[str]) -> str:
     if len(keys) < 3:
-        return keys[0]  # TODO: first vs last occurrence
+        return keys[0] if keys else None  # TODO: first vs last occurrence
     best_count = 0
-    best_key = ''
+    best_key = None
-    tmp = {}  # type: Dict[str, int]
+    tmp = {}  # type: Dict[T, int]
     for k in keys:
         num = (tmp[k] + 1) if k in tmp else 1
         tmp[k] = num
@@ -26,3 +18,45 @@ def most_used_key(keys: List[str]) -> str:
             best_count = num
             best_key = k
     return best_key
+
+
+def split_strip(data: str, delimiter: str = ',') -> List[str]:
+    ''' Split by delimiter and strip each str separately. Omit if empty. '''
+    ret = []
+    for x in data.split(delimiter):
+        x = x.strip()
+        if x:
+            ret.append(x)
+    return ret
+
+
+def insert_before_ext(data: str, ins: str, delimiter: str = '.') -> str:
+    ''' Insert text before last index of delimeter (or at the end). '''
+    assert delimiter in data, 'Could not insert before delimiter: ' + delimiter
+    idx = data.rindex(delimiter)
+    return data[:idx] + ins + data[idx:]
+
+
+def build_url(parts: List[str]) -> str:
+    ''' Build URL similar to lektor.utils.build_url '''
+    url = ''
+    for comp in parts:
+        txt = str(comp).strip('/')
+        if txt:
+            url += '/' + txt
+    if '.' not in url.rsplit('/', 1)[-1]:
+        url += '/'
+    return url or '/'
+
+
+class cached_property(Generic[T]):
+    ''' Calculate complex property only once. '''
+
+    def __init__(self, fn: Callable[[Any], T]) -> None:
+        self.fn = fn
+
+    def __get__(self, obj: object, typ: Union[type, None] = None) -> T:
+        if obj is None:
+            return self  # type: ignore
+        ret = obj.__dict__[self.fn.__name__] = self.fn(obj)
+        return ret

lektor_groupby/vobj.py

@@ -1,12 +1,17 @@
 from lektor.build_programs import BuildProgram  # subclass
 from lektor.context import get_ctx
+from lektor.db import _CmpHelper
 from lektor.environment import Expression
 from lektor.sourceobj import VirtualSourceObject  # subclass
-from lektor.utils import build_url
+from typing import (
-from typing import TYPE_CHECKING, Dict, List, Any, Optional, Iterator
+    TYPE_CHECKING, List, Any, Dict, Optional, Generator, Iterator, Iterable
-from .backref import VGroups
+)
-from .util import report_config_error
+from .pagination import PaginationConfig
+from .pruner import VirtualPruner
+from .query import FixedRecordsQuery
+from .util import most_used_key, insert_before_ext, build_url, cached_property
 if TYPE_CHECKING:
+    from lektor.pagination import Pagination
     from lektor.builder import Artifact
     from lektor.db import Record
     from .config import Config
@@ -22,102 +27,199 @@ class GroupBySource(VirtualSourceObject):
     '''
     Holds information for a single group/cluster.
     This object is accessible in your template file.
-    Attributes: record, key, group, slug, children, config
+    Attributes: record, key, key_obj, slug, children, config
     '''
 
     def __init__(
         self,
         record: 'Record',
-        group: str,
+        key: str,
         config: 'Config',
-        children: Optional[Dict['Record', List[Any]]] = None,
+        page_num: Optional[int] = None
     ) -> None:
         super().__init__(record)
-        self.key = config.slugify(group)
+        self.__children = []  # type: List[str]
-        self.group = group
+        self.__key_obj_map = []  # type: List[Any]
+        self._expr_fields = {}  # type: Dict[str, Expression]
+        self.key = key
         self.config = config
-        self._children = children or {}  # type: Dict[Record, List[Any]]
+        self.page_num = page_num
-        # evaluate slug Expression
-        if config.slug and '{key}' in config.slug:
-            self.slug = config.slug.replace('{key}', self.key)
-        else:
-            self.slug = self._eval(config.slug, field='slug')
-            assert self.slug != Ellipsis, 'invalid config: ' + config.slug
-        if self.slug and self.slug.endswith('/index.html'):
-            self.slug = self.slug[:-10]
-        # extra fields
-        for attr, expr in config.fields.items():
-            setattr(self, attr, self._eval(expr, field='fields.' + attr))
-        # back-ref
-        for child in self._children:
-            VGroups.of(child).add(self)
 
-    def _eval(self, value: Any, *, field: str) -> Any:
+    def append_child(self, child: 'Record', key_obj: Any) -> None:
-        ''' Internal only: evaluates Lektor config file field expression. '''
+        if child.path not in self.__children:
-        if not isinstance(value, str):
+            self.__children.append(child.path)
-            return value
+        # __key_obj_map is later used to find most used key_obj
-        pad = self.record.pad
+        self.__key_obj_map.append(key_obj)
-        alt = self.record.alt
+
-        try:
+    def _update_attr(self, key: str, value: Any) -> None:
-            return Expression(pad.env, value).evaluate(pad, this=self, alt=alt)
+        ''' Set or remove Jinja evaluated Expression field. '''
-        except Exception as e:
+        # TODO: instead we could evaluate the fields only once.
-            report_config_error(self.config.key, field, value, e)
+        #       But then we need to record_dependency() every successive access
-            return Ellipsis
+        if isinstance(value, Expression):
+            self._expr_fields[key] = value
+            try:
+                delattr(self, key)
+            except AttributeError:
+                pass
+        else:
+            if key in self._expr_fields:
+                del self._expr_fields[key]
+            setattr(self, key, value)
+
+    # -------------------------
+    #   Evaluate Extra Fields
+    # -------------------------
+
+    def finalize(self, key_obj: Optional[Any] = None) \
+            -> 'GroupBySource':
+        # make a sorted children query
+        self._query = FixedRecordsQuery(self.pad, self.__children, self.alt)
+        self._query._order_by = self.config.order_by
+        del self.__children
+        # set indexed original value (can be: str, int, float, bool, obj)
+        self.key_obj = key_obj or most_used_key(self.__key_obj_map)
+        del self.__key_obj_map
+
+        if key_obj:  # exit early if initialized through resolver
+            return self
+        # extra fields
+        for attr in self.config.fields:
+            self._update_attr(attr, self.config.eval_field(attr, on=self))
+        return self
+
+    @cached_property
+    def slug(self) -> Optional[str]:
+        # evaluate slug Expression once we need it
+        slug = self.config.eval_slug(self.key, on=self)
+        if slug and slug.endswith('/index.html'):
+            slug = slug[:-10]
+        return slug
+
+    # -----------------------
+    #   Pagination handling
+    # -----------------------
+
+    @property
+    def supports_pagination(self) -> bool:
+        return self.config.pagination['enabled']  # type: ignore[no-any-return]
+
+    @cached_property
+    def _pagination_config(self) -> 'PaginationConfig':
+        # Generate `PaginationConfig` once we need it
+        return PaginationConfig(self.record.pad.env, self.config.pagination,
+                                self._query.total)
+
+    @cached_property
+    def pagination(self) -> 'Pagination':
+        # Generate `Pagination` once we need it
+        return self._pagination_config.get_pagination_controller(self)
+
+    def __iter_pagination_sources__(self) -> Iterator['GroupBySource']:
+        ''' If pagination enabled, yields `GroupBySourcePage` sub-pages. '''
+        # Used in GroupBy.make_once() to generated paginated child sources
+        if self._pagination_config.enabled and self.page_num is None:
+            for page_num in range(self._pagination_config.count_pages(self)):
+                yield self.__for_page__(page_num + 1)
+
+    def __for_page__(self, page_num: Optional[int]) -> 'GroupBySource':
+        ''' Get source object for a (possibly) different page number. '''
+        assert page_num is not None
+        return GroupBySourcePage(self, page_num)
 
     # ---------------------
     #   Lektor properties
     # ---------------------
 
     @property
-    def path(self) -> str:
+    def path(self) -> str:  # type: ignore[override]
         # Used in VirtualSourceInfo, used to prune VirtualObjects
-        return f'{self.record.path}{VPATH}/{self.config.key}/{self.key}'
+        vpath = f'{self.record.path}{VPATH}/{self.config.key}/{self.key}'
+        if self.page_num:
+            vpath += '/' + str(self.page_num)
+        return vpath
 
-    @property
+    @cached_property
-    def url_path(self) -> str:
+    def url_path(self) -> str:  # type: ignore[override]
-        # Actual path to resource as seen by the browser
+        ''' Actual path to resource as seen by the browser. '''
-        return build_url([self.record.path, self.slug])  # slug can be None!
+        # check if slug is absolute URL
+        slug = self.slug
+        if slug and slug.startswith('/'):
+            parts = [self.pad.get_root(alt=self.alt).url_path]
+        else:
+            parts = [self.record.url_path]
+        # slug can be None!!
+        if not slug:
+            return build_url(parts)
+        # if pagination enabled, append pagination.url_suffix to path
+        if self.page_num and self.page_num > 1:
+            sffx = self._pagination_config.url_suffix
+            if '.' in slug.rsplit('/', 1)[-1]:
+                # default: ../slugpage2.html (use e.g.: url_suffix = .page.)
+                parts.append(insert_before_ext(
+                    slug, sffx + str(self.page_num), '.'))
+            else:
+                # default: ../slug/page/2/index.html
+                parts += [slug, sffx, self.page_num]
+        else:
+            parts.append(slug)
+        return build_url(parts)
 
-    def iter_source_filenames(self) -> Iterator[str]:
+    def iter_source_filenames(self) -> Generator[str, None, None]:
         ''' Enumerate all dependencies '''
         if self.config.dependencies:
             yield from self.config.dependencies
-        for record in self._children:
+        for record in self.children:
             yield from record.iter_source_filenames()
 
+    # def get_checksum(self, path_cache: 'PathCache') -> Optional[str]:
+    #     deps = [self.pad.env.jinja_env.get_or_select_template(
+    #         self.config.template).filename]
+    #     deps.extend(self.iter_source_filenames())
+    #     sums = '|'.join(path_cache.get_file_info(x).filename_and_checksum
+    #                     for x in deps if x) + str(self.children.count())
+    #     return hashlib.sha1(sums.encode('utf-8')).hexdigest() if sums else None
+
+    def get_sort_key(self, fields: Iterable[str]) -> List:
+        def cmp_val(field: str) -> Any:
+            reverse = field.startswith('-')
+            if reverse or field.startswith('+'):
+                field = field[1:]
+            return _CmpHelper(getattr(self, field, None), reverse)
+
+        return [cmp_val(field) for field in fields or []]
+
     # -----------------------
     #   Properties & Helper
     # -----------------------
 
     @property
-    def children(self) -> Dict['Record', List[Any]]:
+    def children(self) -> FixedRecordsQuery:
-        ''' Returns dict with page record key and (optional) extra value. '''
+        ''' Return query of children of type Record. '''
-        return self._children
+        return self._query
-
-    @property
-    def first_child(self) -> Optional['Record']:
-        ''' Returns first referencing page record. '''
-        if self._children:
-            return iter(self._children).__next__()
-        return None
-
-    @property
-    def first_extra(self) -> Optional[Any]:
-        ''' Returns first additional / extra info object of first page. '''
-        if not self._children:
-            return None
-        val = iter(self._children.values()).__next__()
-        return val[0] if val else None
 
     def __getitem__(self, key: str) -> Any:
         # Used for virtual path resolver
         if key in ('_path', '_alt'):
             return getattr(self, key[1:])
-        return self.__missing__(key)  # type: ignore[attr-defined]
+        return self.__missing__(key)
+
+    def __getattr__(self, key: str) -> Any:
+        ''' Lazy evaluate custom user field expressions. '''
+        if key in self._expr_fields:
+            expr = self._expr_fields[key]
+            return expr.evaluate(self.pad, this=self, alt=self.alt)
+        raise AttributeError
 
     def __lt__(self, other: 'GroupBySource') -> bool:
-        # Used for |sort filter ("group" is the provided original string)
+        # Used for |sort filter (`key_obj` is the indexed original value)
-        return self.group.lower() < other.group.lower()
+        if isinstance(self.key_obj, (bool, int, float)) and \
+                isinstance(other.key_obj, (bool, int, float)):
+            return self.key_obj < other.key_obj
+        if self.key_obj is None:
+            return False  # this will sort None at the end
+        if other.key_obj is None:
+            return True
+        return str(self.key_obj).lower() < str(other.key_obj).lower()
 
     def __eq__(self, other: object) -> bool:
         # Used for |unique filter
@@ -132,7 +234,8 @@ class GroupBySource(VirtualSourceObject):
 
     def __repr__(self) -> str:
         return '<GroupBySource path="{}" children={}>'.format(
-            self.path, len(self._children))
+            self.path,
+            self.children.count() if hasattr(self, 'children') else '?')
 
 
 # -----------------------------------
@@ -143,6 +246,9 @@ class GroupByBuildProgram(BuildProgram):
     ''' Generate Build-Artifacts and write files. '''
 
     def produce_artifacts(self) -> None:
+        pagination_enabled = self.source._pagination_config.enabled
+        if pagination_enabled and self.source.page_num is None:
+            return  # only __iter_pagination_sources__()
         url = self.source.url_path
         if url.endswith('/'):
             url += 'index.html'
@@ -150,6 +256,32 @@ class GroupByBuildProgram(BuildProgram):
             self.source.iter_source_filenames()))
 
     def build_artifact(self, artifact: 'Artifact') -> None:
-        get_ctx().record_virtual_dependency(self.source)
+        get_ctx().record_virtual_dependency(VirtualPruner())
         artifact.render_template_into(
             self.source.config.template, this=self.source)
+
+
+class GroupBySourcePage(GroupBySource):
+    ''' Pagination wrapper. Redirects get attr/item to non-paginated node. '''
+
+    def __init__(self, parent: 'GroupBySource', page_num: int) -> None:
+        self.__parent = parent
+        self.page_num = page_num
+
+    def __for_page__(self, page_num: Optional[int]) -> 'GroupBySource':
+        ''' Get source object for a (possibly) different page number. '''
+        if page_num is None:
+            return self.__parent
+        if page_num == self.page_num:
+            return self
+        return GroupBySourcePage(self.__parent, page_num)
+
+    def __getitem__(self, key: str) -> Any:
+        return self.__parent.__getitem__(key)
+
+    def __getattr__(self, key: str) -> Any:
+        return getattr(self.__parent, key)
+
+    def __repr__(self) -> str:
+        return '<GroupBySourcePage path="{}" page={}>'.format(
+            self.__parent.path, self.page_num)

lektor_groupby/watcher.py

@@ -1,10 +1,12 @@
-from typing import TYPE_CHECKING, Dict, List, Tuple, Any, Union, NamedTuple
+from typing import (
-from typing import Optional, Callable, Iterator, Generator
+    TYPE_CHECKING, Dict, List, Any, Union, NamedTuple,
+    Optional, Callable, Iterator, Generator
+)
+from .backref import VGroups
 from .model import ModelReader
-from .util import most_used_key
 from .vobj import GroupBySource
 if TYPE_CHECKING:
-    from lektor.db import Database, Record
+    from lektor.db import Pad, Record
     from .config import Config
     from .model import FieldKeyPath
 
@@ -16,15 +18,15 @@ class GroupByCallbackArgs(NamedTuple):
 
 
 GroupingCallback = Callable[[GroupByCallbackArgs], Union[
-    Iterator[Union[str, Tuple[str, Any]]],
+    Iterator[Any],
-    Generator[Union[str, Tuple[str, Any]], Optional[str], None],
+    Generator[Any, Optional[GroupBySource], None],
 ]]
 
 
 class Watcher:
     '''
     Callback is called with (Record, FieldKeyPath, field-value).
-    Callback may yield one or more (group, extra-info) tuples.
+    Callback may yield 0-n objects.
     '''
 
     def __init__(self, config: 'Config') -> None:
@@ -37,23 +39,27 @@ class Watcher:
         Decorator to subscribe to attrib-elements.
         If flatten = False, dont explode FlowType.
 
-        (record, field-key, field) -> (group, extra-info)
+        (record, field-key, field) -> value
         '''
         def _decorator(fn: GroupingCallback) -> None:
             self.flatten = flatten
             self.callback = fn
         return _decorator
 
-    def initialize(self, db: 'Database') -> None:
+    def initialize(self, pad: 'Pad') -> None:
         ''' Reset internal state. You must initialize before each build! '''
         assert callable(self.callback), 'No grouping callback provided.'
-        self._model_reader = ModelReader(db, self.config.key, self.flatten)
+        self._model_reader = ModelReader(pad.db, self.config.key, self.flatten)
-        self._state = {}  # type: Dict[str, Dict[Record, List[Any]]]
+        self._root_record = {}  # type: Dict[str, Record]
-        self._group_map = {}  # type: Dict[str, List[str]]
+        self._state = {}  # type: Dict[str, Dict[Optional[str], GroupBySource]]
+        self._rmmbr = []  # type: List[Record]
+        for alt in pad.config.iter_alternatives():
+            self._root_record[alt] = pad.get(self._root, alt=alt)
+            self._state[alt] = {}
 
     def should_process(self, node: 'Record') -> bool:
         ''' Check if record path is being watched. '''
-        return node['_path'].startswith(self._root)
+        return str(node['_path']).startswith(self._root)
 
     def process(self, record: 'Record') -> None:
         '''
@@ -61,56 +67,82 @@ class Watcher:
         Each record is guaranteed to be processed only once.
         '''
         for key, field in self._model_reader.read(record):
-            _gen = self.callback(GroupByCallbackArgs(record, key, field))
+            args = GroupByCallbackArgs(record, key, field)
+            _gen = self.callback(args)
             try:
-                obj = next(_gen)
+                key_obj = next(_gen)
                 while True:
-                    if not isinstance(obj, (str, tuple)):
+                    if self.config.key_obj_fn:
-                        raise TypeError(f'Unsupported groupby yield: {obj}')
+                        vobj = self._persist_multiple(args, key_obj)
-                    slug = self._persist(record, key, obj)
-                    # return slugified group key and continue iteration
-                    if isinstance(_gen, Generator) and not _gen.gi_yieldfrom:
-                        obj = _gen.send(slug)
                     else:
-                        obj = next(_gen)
+                        vobj = self._persist(args, key_obj)
+                    # return groupby virtual object and continue iteration
+                    if isinstance(_gen, Generator) and not _gen.gi_yieldfrom:
+                        key_obj = _gen.send(vobj)
+                    else:
+                        key_obj = next(_gen)
             except StopIteration:
                 del _gen
 
-    def _persist(
+    def _persist_multiple(self, args: 'GroupByCallbackArgs', obj: Any) \
-        self,
+            -> Optional[GroupBySource]:
-        record: 'Record',
+        # if custom key mapping function defined, use that first
-        key: 'FieldKeyPath',
+        res = self.config.eval_key_obj_fn(on=args.record,
-        obj: Union[str, tuple]
+                                          context={'X': obj, 'ARGS': args})
-    ) -> str:
+        if isinstance(res, (list, tuple)):
-        ''' Update internal state. Return slugified string. '''
+            for k in res:
-        group = obj if isinstance(obj, str) else obj[0]
+                self._persist(args, k)  # 1-to-n replacement
-        slug = self.config.slugify(group)
+            return None
-        # init group-key
+        return self._persist(args, res)  # normal & null replacement
-        if slug not in self._state:
-            self._state[slug] = {}
-            self._group_map[slug] = []
-        # _group_map is later used to find most used group
-        self._group_map[slug].append(group)
-        # init group extras
-        if record not in self._state[slug]:
-            self._state[slug][record] = []
-        # append extras (or default value)
-        if isinstance(obj, tuple):
-            self._state[slug][record].append(obj[1])
-        else:
-            self._state[slug][record].append(key.fieldKey)
-        return slug
 
-    def iter_sources(self, root: 'Record') -> Iterator[GroupBySource]:
+    def _persist(self, args: 'GroupByCallbackArgs', obj: Any) \
+            -> Optional[GroupBySource]:
+        ''' Update internal state. Return grouping parent. '''
+        if not isinstance(obj, (str, bool, int, float)) and obj is not None:
+            raise ValueError(
+                'Unsupported groupby yield type for [{}]:'
+                ' {} (expected str, got {})'.format(
+                    self.config.key, obj, type(obj).__name__))
+
+        if obj is None:
+            # if obj is not set, test if config.replace_none_key is set
+            slug = self.config.replace_none_key
+            obj = slug
+        else:
+            # if obj is set, apply config.key_map  (convert int -> str)
+            slug = self.config.slugify(str(obj)) or None
+        # if neither custom mapping succeeded, do not process further
+        if not slug or obj is None:
+            return None
+        # update internal object storage
+        alt = args.record.alt
+        if slug not in self._state[alt]:
+            src = GroupBySource(self._root_record[alt], slug, self.config)
+            self._state[alt][slug] = src
+        else:
+            src = self._state[alt][slug]
+
+        src.append_child(args.record, obj)
+        # reverse reference
+        VGroups.of(args.record).add(args.key, src)
+        return src
+
+    def remember(self, record: 'Record') -> None:
+        self._rmmbr.append(record)
+
+    def iter_sources(self) -> Iterator[GroupBySource]:
         ''' Prepare and yield GroupBySource elements. '''
-        for key, children in self._state.items():
+        for x in self._rmmbr:
-            group = most_used_key(self._group_map[key])
+            self.process(x)
-            yield GroupBySource(root, group, self.config, children=children)
+        del self._rmmbr
+        for vobj_list in self._state.values():
+            for vobj in vobj_list.values():
+                yield vobj.finalize()
         # cleanup. remove this code if you'd like to iter twice
         del self._model_reader
+        del self._root_record
         del self._state
-        del self._group_map
 
     def __repr__(self) -> str:
-        return '<GroupByWatcher key="{}" enabled={} callback={}>'.format(
+        return '<GroupByWatcher key="{}" enabled={}>'.format(
-            self.config.key, self.config.enabled, self.callback)
+            self.config.key, self.config.enabled)

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup
 
-with open('README.md') as fp:
+with open('README.md', encoding='utf8') as fp:
     longdesc = fp.read()
 
 setup(
@@ -13,7 +13,7 @@ setup(
     },
     author='relikd',
     url='https://github.com/relikd/lektor-groupby-plugin',
-    version='0.9.6',
+    version='1.0.0',
     description='Cluster arbitrary records with field attribute keyword.',
     long_description=longdesc,
     long_description_content_type="text/markdown",