diff --git a/lektor_groupby/backref.py b/lektor_groupby/backref.py index eb194b0..6a1bba9 100644 --- a/lektor_groupby/backref.py +++ b/lektor_groupby/backref.py @@ -53,16 +53,6 @@ class VGroups: order_by: Union[str, Iterable[str], None] = None, ) -> Iterator['GroupBySource']: ''' Extract all referencing groupby virtual objects from a page. ''' - ctx = get_ctx() - if not ctx: - raise NotImplementedError("Shouldn't happen, where is my context?") - # get GroupBy object - builder = ctx.build_state.builder - groupby = GroupByRef.of(builder) - groupby.make_once(builder) # ensure did cluster before - # manage config dependencies - for dep in groupby.dependencies: - ctx.record_dependency(dep) # prepare filter if isinstance(keys, str): keys = [keys] @@ -70,6 +60,13 @@ class VGroups: fields = [fields] if isinstance(flows, str): flows = [flows] + # get GroupBy object + ctx = get_ctx() + if not ctx: + raise NotImplementedError("Shouldn't happen, where is my context?") + builder = ctx.build_state.builder + # TODO: fix record_dependency -> process in non-capturing context + GroupByRef.of(builder).make_once(keys) # ensure did cluster before use # find groups proc_list = [record] done_list = set() # type: Set[GroupBySource] @@ -86,6 +83,13 @@ class VGroups: continue done_list.add(vobj()) + # manage config dependencies + deps = set() # type: Set[str] + for vobj in done_list: + deps.update(vobj.config.dependencies) + for dep in deps: + ctx.record_dependency(dep) + if order_by: if isinstance(order_by, str): order = split_strip(order_by, ',') # type: Iterable[str] diff --git a/lektor_groupby/config.py b/lektor_groupby/config.py index fe9c371..85a831d 100644 --- a/lektor_groupby/config.py +++ b/lektor_groupby/config.py @@ -1,12 +1,33 @@ from inifile import IniFile -from lektor.utils import slugify +from lektor.environment import Expression +from lektor.context import Context +from lektor.utils import slugify as _slugify +from typing import TYPE_CHECKING +from typing import Set, Dict, Optional, Union, Any, List, Generator from .util import split_strip +if TYPE_CHECKING: + from lektor.sourceobj import SourceObject -from typing import Set, Dict, Optional, Union, Any, List AnyConfig = Union['Config', IniFile, Dict] +class ConfigError(Exception): + ''' Used to print a Lektor console error. ''' + + def __init__( + self, key: str, field: str, expr: str, error: Union[Exception, str] + ): + self.key = key + self.field = field + self.expr = expr + self.error = error + + def __str__(self) -> str: + return 'Invalid config for [{}.{}] = "{}" – Error: {}'.format( + self.key, self.field, self.expr, repr(self.error)) + + class Config: ''' Holds information for GroupByWatcher and GroupBySource. @@ -22,11 +43,15 @@ class Config: root: Optional[str] = None, # default: "/" slug: Optional[str] = None, # default: "{attr}/{group}/index.html" template: Optional[str] = None, # default: "groupby-{attr}.html" + replace_none_key: Optional[str] = None, # default: None + key_map_fn: Optional[str] = None, # default: None ) -> None: self.key = key self.root = (root or '/').rstrip('/') or '/' self.slug = slug or (key + '/{key}/') # key = GroupBySource.key self.template = template or f'groupby-{self.key}.html' + self.replace_none_key = replace_none_key + self.key_map_fn = key_map_fn # editable after init self.enabled = True self.dependencies = set() # type: Set[str] @@ -37,7 +62,8 @@ class Config: def slugify(self, k: str) -> str: ''' key_map replace and slugify. ''' - return slugify(self.key_map.get(k, k)) # type: ignore[no-any-return] + rv = self.key_map.get(k, k) + return _slugify(rv) or rv # the `or` allows for example "_" def set_fields(self, fields: Optional[Dict[str, Any]]) -> None: ''' @@ -72,7 +98,7 @@ class Config: def __repr__(self) -> str: txt = ' Union[Expression, Any]: + ''' Create Expression and report any config error. ''' + if not isinstance(expr, str): + return expr + try: + return Expression(on.pad.env, expr) + except Exception as e: + raise ConfigError(self.key, field, expr, e) + + def eval_field(self, attr: str, *, on: 'SourceObject') \ + -> Union[Expression, Any]: + ''' Create an expression for a custom defined user field. ''' + # do not `gather_dependencies` because fields are evaluated on the fly + # dependency tracking happens whenever a field is accessed + return self._make_expression( + self.fields[attr], on=on, field='fields.' + attr) + + def eval_slug(self, key: str, *, on: 'SourceObject') -> Optional[str]: + ''' Either perform a "{key}" substitution or evaluate expression. ''' + cfg_slug = self.slug + if not cfg_slug: + return None + if '{key}' in cfg_slug: + if key: + return cfg_slug.replace('{key}', key) + else: + raise ConfigError(self.key, 'slug', cfg_slug, + 'Cannot replace {key} with None') + return None + else: + # TODO: do we need `gather_dependencies` here too? + expr = self._make_expression(cfg_slug, on=on, field='slug') + return expr.evaluate(on.pad, this=on, alt=on.alt) or None + + def eval_key_map_fn(self, *, on: 'SourceObject', context: Dict) -> Any: + ''' + If `key_map_fn` is set, evaluate field expression. + Note: The function does not check whether `key_map_fn` is set. + Return: A Generator result is automatically unpacked into a list. + ''' + exp = self._make_expression(self.key_map_fn, on=on, field='key_map_fn') + with Context(pad=on.pad) as ctx: + with ctx.gather_dependencies(self.dependencies.add): + res = exp.evaluate(on.pad, this=on, alt=on.alt, values=context) + if isinstance(res, Generator): + res = list(res) # unpack for 1-to-n replacement + return res diff --git a/lektor_groupby/groupby.py b/lektor_groupby/groupby.py index 2cd01f2..c046568 100644 --- a/lektor_groupby/groupby.py +++ b/lektor_groupby/groupby.py @@ -1,6 +1,7 @@ from lektor.builder import PathCache from lektor.db import Record # isinstance -from typing import TYPE_CHECKING, Set, List +from lektor.reporter import reporter # build +from typing import TYPE_CHECKING, List, Optional, Iterable from .config import Config from .watcher import Watcher if TYPE_CHECKING: @@ -19,14 +20,14 @@ class GroupBy: ''' def __init__(self, resolver: 'Resolver') -> None: + self._building = False self._watcher = [] # type: List[Watcher] self._results = [] # type: List[GroupBySource] self.resolver = resolver - self.didBuild = False @property - def isNew(self) -> bool: - return not self.didBuild + def isBuilding(self) -> bool: + return self._building def add_watcher(self, key: str, config: 'AnyConfig') -> Watcher: ''' Init Config and add to watch list. ''' @@ -34,15 +35,8 @@ class GroupBy: self._watcher.append(w) return w - def get_dependencies(self) -> Set[str]: - deps = set() # type: Set[str] - for w in self._watcher: - deps.update(w.config.dependencies) - return deps - def queue_all(self, builder: 'Builder') -> None: ''' Iterate full site-tree and queue all children. ''' - self.dependencies = self.get_dependencies() # remove disabled watchers self._watcher = [w for w in self._watcher if w.config.enabled] if not self._watcher: @@ -61,31 +55,58 @@ class GroupBy: if isinstance(record, Record): for w in self._watcher: if w.should_process(record): - w.process(record) + w.remember(record) - def make_once(self, builder: 'Builder') -> None: - ''' Perform groupby, iter over sources with watcher callback. ''' - self.didBuild = True - if self._watcher: + def make_once(self, filter_keys: Optional[Iterable[str]] = None) -> None: + ''' + Perform groupby, iter over sources with watcher callback. + If `filter_keys` is set, ignore all other watchers. + ''' + if not self._watcher: + return + # not really necessary but should improve performance of later reset() + if not filter_keys: self.resolver.reset() - for w in self._watcher: - root = builder.pad.get(w.config.root) - for vobj in w.iter_sources(root): - # add original source - self._results.append(vobj) - self.resolver.add(vobj) - # and also add pagination sources - for sub_vobj in vobj.__iter_pagination_sources__(): - self._results.append(sub_vobj) - self.resolver.add(sub_vobj) - self._watcher.clear() + remaining = [] + for w in self._watcher: + # only process vobjs that are used somewhere + if filter_keys and w.config.key not in filter_keys: + remaining.append(w) + continue + self.resolver.reset(w.config.key) + # these are used in the current context (or on `build_all`) + for vobj in w.iter_sources(): + # add original source + self._results.append(vobj) + self.resolver.add(vobj) + # and also add pagination sources + for sub_vobj in vobj.__iter_pagination_sources__(): + self._results.append(sub_vobj) + self.resolver.add(sub_vobj) + # TODO: if this should ever run concurrently, pop() from watchers + self._watcher = remaining - def build_all(self, builder: 'Builder') -> None: - ''' Create virtual objects and build sources. ''' - self.make_once(builder) # in case no page used the |vgroups filter - path_cache = PathCache(builder.env) - for vobj in self._results: - if vobj.slug: - builder.build(vobj, path_cache) - del path_cache - self._results.clear() # garbage collect weak refs + def build_all( + self, + builder: 'Builder', + specific: Optional['GroupBySource'] = None + ) -> None: + ''' + Build actual artifacts (if needed). + If `specific` is set, only build the artifacts for that single vobj + ''' + if not self._watcher and not self._results: + return + with reporter.build('groupby', builder): # type:ignore + # in case no page used the |vgroups filter + self.make_once([specific.config.key] if specific else None) + self._building = True + path_cache = PathCache(builder.env) + for vobj in self._results: + if specific and vobj.path != specific.path: + continue + if vobj.slug: + builder.build(vobj, path_cache) + del path_cache + self._building = False + self._results.clear() # garbage collect weak refs diff --git a/lektor_groupby/model.py b/lektor_groupby/model.py index 1d45f06..286e93e 100644 --- a/lektor_groupby/model.py +++ b/lektor_groupby/model.py @@ -48,6 +48,7 @@ class ModelReader: for r_key, subs in self._models.get(record.datamodel.id, {}).items(): field = record[r_key] if not field: + yield FieldKeyPath(r_key), field continue if subs == '*': # either normal field or flow type (all blocks) if self.flatten and isinstance(field, Flow): diff --git a/lektor_groupby/plugin.py b/lektor_groupby/plugin.py index 25af87e..6ee93b3 100644 --- a/lektor_groupby/plugin.py +++ b/lektor_groupby/plugin.py @@ -7,7 +7,7 @@ from .pruner import prune from .resolver import Resolver from .vobj import VPATH, GroupBySource, GroupByBuildProgram if TYPE_CHECKING: - from lektor.builder import Builder, BuildState + from lektor.builder import Builder from lektor.sourceobj import SourceObject from .watcher import GroupByCallbackArgs @@ -17,7 +17,6 @@ class GroupByPlugin(Plugin): description = 'Cluster arbitrary records with field attribute keyword.' def on_setup_env(self, **extra: Any) -> None: - self.has_changes = False self.resolver = Resolver(self.env) self.env.add_build_program(GroupBySource, GroupByBuildProgram) self.env.jinja_env.filters.update(vgroups=VGroups.iter) @@ -30,19 +29,13 @@ class GroupByPlugin(Plugin): if isinstance(source, Asset): return groupby = self._init_once(builder) - if groupby.isNew and isinstance(source, GroupBySource): - self.has_changes = True - - def on_after_build(self, build_state: 'BuildState', **extra: Any) -> None: - if build_state.updated_artifacts: - self.has_changes = True + if not groupby.isBuilding and isinstance(source, GroupBySource): + # TODO: differentiate between actual build and browser preview + groupby.build_all(builder, source) def on_after_build_all(self, builder: 'Builder', **extra: Any) -> None: - # only rebuild if has changes (bypass idle builds) - # or the very first time after startup (url resolver & pruning) - if self.has_changes or not self.resolver.has_any: - self._init_once(builder).build_all(builder) # updates resolver - self.has_changes = False + # by now, most likely already built. So, build_all() is a no-op + self._init_once(builder).build_all(builder) def on_after_prune(self, builder: 'Builder', **extra: Any) -> None: # TODO: find a better way to prune unreferenced elements @@ -78,7 +71,11 @@ class GroupByPlugin(Plugin): @watcher.grouping() def _fn(args: 'GroupByCallbackArgs') -> Iterator[str]: val = args.field - if isinstance(val, str): + if isinstance(val, str) and val != '': val = map(str.strip, val.split(split)) if split else [val] + elif isinstance(val, (bool, int, float)): + val = [val] + elif not val: # after checking for '', False, 0, and 0.0 + val = [None] if isinstance(val, (list, map)): yield from val diff --git a/lektor_groupby/resolver.py b/lektor_groupby/resolver.py index ca459d7..e8bde5f 100644 --- a/lektor_groupby/resolver.py +++ b/lektor_groupby/resolver.py @@ -42,9 +42,13 @@ class Resolver: def files(self) -> Iterable[str]: return self._data - def reset(self) -> None: + def reset(self, optional_key: Optional[str] = None) -> None: ''' Clear previously recorded virtual objects. ''' - self._data.clear() + if optional_key: + self._data = {k: v for k, v in self._data.items() + if v.config.key != optional_key} + else: + self._data.clear() def add(self, vobj: GroupBySource) -> None: ''' Track new virtual object (only if slug is set). ''' diff --git a/lektor_groupby/util.py b/lektor_groupby/util.py index 3f90824..0346669 100644 --- a/lektor_groupby/util.py +++ b/lektor_groupby/util.py @@ -1,20 +1,9 @@ -from lektor.reporter import reporter, style from typing import List, Dict, Optional, TypeVar from typing import Callable, Any, Union, Generic T = TypeVar('T') -def report_config_error(key: str, field: str, val: str, e: Exception) -> None: - ''' Send error message to Lektor reporter. Indicate which field is bad. ''' - msg = '[ERROR] invalid config for [{}.{}] = "{}", Error: {}'.format( - key, field, val, repr(e)) - try: - reporter._write_line(style(msg, fg='red')) - except Exception: - print(msg) # fallback in case Lektor API changes - - def most_used_key(keys: List[T]) -> Optional[T]: ''' Find string with most occurrences. ''' if len(keys) < 3: @@ -58,3 +47,16 @@ def build_url(parts: List[str]) -> str: if '.' not in url.rsplit('/', 1)[-1]: url += '/' return url or '/' + + +class cached_property(Generic[T]): + ''' Calculate complex property only once. ''' + + def __init__(self, fn: Callable[[Any], T]) -> None: + self.fn = fn + + def __get__(self, obj: object, typ: Union[type, None] = None) -> T: + if obj is None: + return self # type: ignore + ret = obj.__dict__[self.fn.__name__] = self.fn(obj) + return ret diff --git a/lektor_groupby/vobj.py b/lektor_groupby/vobj.py index d455427..16ae905 100644 --- a/lektor_groupby/vobj.py +++ b/lektor_groupby/vobj.py @@ -3,14 +3,11 @@ from lektor.context import get_ctx from lektor.db import _CmpHelper from lektor.environment import Expression from lektor.sourceobj import VirtualSourceObject # subclass -from werkzeug.utils import cached_property - -from typing import TYPE_CHECKING, List, Any, Optional, Iterator, Iterable +from typing import TYPE_CHECKING +from typing import List, Any, Dict, Optional, Generator, Iterator, Iterable from .pagination import PaginationConfig from .query import FixedRecordsQuery -from .util import ( - report_config_error, most_used_key, insert_before_ext, build_url -) +from .util import most_used_key, insert_before_ext, build_url, cached_property if TYPE_CHECKING: from lektor.pagination import Pagination from lektor.builder import Artifact @@ -40,60 +37,64 @@ class GroupBySource(VirtualSourceObject): super().__init__(record) self.key = slug self.page_num = page_num + self._expr_fields = {} # type: Dict[str, Expression] self.__children = [] # type: List[str] - self.__group_map = [] # type: List[str] + self.__group_map = [] # type: List[Any] - def append_child(self, child: 'Record', group: str) -> None: + def append_child(self, child: 'Record', group: Any) -> None: if child not in self.__children: self.__children.append(child.path) + # TODO: rename group to value # __group_map is later used to find most used group self.__group_map.append(group) + def _update_attr(self, key: str, value: Any) -> None: + ''' Set or remove Jinja evaluated Expression field. ''' + if isinstance(value, Expression): + self._expr_fields[key] = value + try: + delattr(self, key) + except AttributeError: + pass + else: + if key in self._expr_fields: + del self._expr_fields[key] + setattr(self, key, value) + # ------------------------- # Evaluate Extra Fields # ------------------------- - def finalize(self, config: 'Config', group: Optional[str] = None) \ + def finalize(self, config: 'Config', group: Optional[Any] = None) \ -> 'GroupBySource': self.config = config # make a sorted children query self._query = FixedRecordsQuery(self.pad, self.__children, self.alt) self._query._order_by = config.order_by + del self.__children # set group name self.group = group or most_used_key(self.__group_map) - # cleanup temporary data - del self.__children del self.__group_map # evaluate slug Expression - self.slug = None # type: Optional[str] - if config.slug and '{key}' in config.slug: - self.slug = config.slug.replace('{key}', self.key) - else: - self.slug = self._eval(config.slug, field='slug') - assert self.slug != Ellipsis, 'invalid config: ' + config.slug + self.slug = config.eval_slug(self.key, on=self) if self.slug and self.slug.endswith('/index.html'): self.slug = self.slug[:-10] - # extra fields - for attr, expr in config.fields.items(): - setattr(self, attr, self._eval(expr, field='fields.' + attr)) - return self - def _eval(self, value: Any, *, field: str) -> Any: - ''' Internal only: evaluates Lektor config file field expression. ''' - if not isinstance(value, str): - return value - pad = self.record.pad - alt = self.record.alt - try: - return Expression(pad.env, value).evaluate(pad, this=self, alt=alt) - except Exception as e: - report_config_error(self.config.key, field, value, e) - return Ellipsis + if group: # exit early if initialized through resolver + return self + # extra fields + for attr in config.fields: + self._update_attr(attr, config.eval_field(attr, on=self)) + return self # ----------------------- # Pagination handling # ----------------------- + @property + def supports_pagination(self) -> bool: + return self.config.pagination['enabled'] # type: ignore[no-any-return] + @cached_property def _pagination_config(self) -> 'PaginationConfig': # Generate `PaginationConfig` once we need it @@ -128,25 +129,30 @@ class GroupBySource(VirtualSourceObject): vpath += '/' + str(self.page_num) return vpath - @property - def url_path(self) -> str: - # Actual path to resource as seen by the browser - parts = [self.record.url_path] + @cached_property + def url_path(self) -> str: # type: ignore[override] + ''' Actual path to resource as seen by the browser. ''' + # check if slug is absolute URL + slug = self.slug + if slug and slug.startswith('/'): + parts = [self.pad.get_root(alt=self.alt).url_path] + else: + parts = [self.record.url_path] # slug can be None!! - if not self.slug: + if not slug: return build_url(parts) # if pagination enabled, append pagination.url_suffix to path if self.page_num and self.page_num > 1: sffx = self._pagination_config.url_suffix - if '.' in self.slug.split('/')[-1]: + if '.' in slug.rsplit('/', 1)[-1]: # default: ../slugpage2.html (use e.g.: url_suffix = .page.) parts.append(insert_before_ext( - self.slug, sffx + str(self.page_num), '.')) + slug, sffx + str(self.page_num), '.')) else: # default: ../slug/page/2/index.html - parts += [self.slug, sffx, self.page_num] + parts += [slug, sffx, self.page_num] else: - parts.append(self.slug) + parts.append(slug) return build_url(parts) def iter_source_filenames(self) -> Generator[str, None, None]: @@ -188,9 +194,23 @@ class GroupBySource(VirtualSourceObject): return getattr(self, key[1:]) return self.__missing__(key) + def __getattr__(self, key: str) -> Any: + ''' Lazy evaluate custom user field expressions. ''' + if key in self._expr_fields: + expr = self._expr_fields[key] + return expr.evaluate(self.pad, this=self, alt=self.alt) + raise AttributeError + def __lt__(self, other: 'GroupBySource') -> bool: # Used for |sort filter ("group" is the provided original string) - return self.group.lower() < other.group.lower() + if isinstance(self.group, (bool, int, float)) and \ + isinstance(other.group, (bool, int, float)): + return self.group < other.group + if self.group is None: + return False + if other.group is None: + return True + return str(self.group).lower() < str(other.group).lower() def __eq__(self, other: object) -> bool: # Used for |unique filter diff --git a/lektor_groupby/watcher.py b/lektor_groupby/watcher.py index 0e32d8f..763ea9b 100644 --- a/lektor_groupby/watcher.py +++ b/lektor_groupby/watcher.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, List, Tuple, Any, Union, NamedTuple +from typing import TYPE_CHECKING, Dict, List, Any, Union, NamedTuple from typing import Optional, Callable, Iterator, Generator from .backref import VGroups from .model import ModelReader @@ -16,8 +16,8 @@ class GroupByCallbackArgs(NamedTuple): GroupingCallback = Callable[[GroupByCallbackArgs], Union[ - Iterator[Union[str, Tuple[str, Any]]], - Generator[Union[str, Tuple[str, Any]], Optional[str], None], + Iterator[Any], + Generator[Optional[str], Any, None], ]] @@ -49,7 +49,8 @@ class Watcher: assert callable(self.callback), 'No grouping callback provided.' self._model_reader = ModelReader(pad.db, self.config.key, self.flatten) self._root_record = {} # type: Dict[str, Record] - self._state = {} # type: Dict[str, Dict[str, GroupBySource]] + self._state = {} # type: Dict[str, Dict[Optional[str], GroupBySource]] + self._rmmbr = [] # type: List[Record] for alt in pad.config.iter_alternatives(): self._root_record[alt] = pad.get(self._root, alt=alt) self._state[alt] = {} @@ -64,13 +65,15 @@ class Watcher: Each record is guaranteed to be processed only once. ''' for key, field in self._model_reader.read(record): - _gen = self.callback(GroupByCallbackArgs(record, key, field)) + args = GroupByCallbackArgs(record, key, field) + _gen = self.callback(args) try: group = next(_gen) while True: - if not isinstance(group, str): - raise TypeError(f'Unsupported groupby yield: {group}') - slug = self._persist(record, key, group) + if self.config.key_map_fn: + slug = self._persist_multiple(args, group) + else: + slug = self._persist(args, group) # return slugified group key and continue iteration if isinstance(_gen, Generator) and not _gen.gi_yieldfrom: group = _gen.send(slug) @@ -79,24 +82,57 @@ class Watcher: except StopIteration: del _gen - def _persist(self, record: 'Record', key: 'FieldKeyPath', group: str) \ - -> str: + def _persist_multiple(self, args: 'GroupByCallbackArgs', obj: Any) \ + -> Optional[str]: + # if custom key mapping function defined, use that first + res = self.config.eval_key_map_fn(on=args.record, + context={'X': obj, 'SRC': args}) + if isinstance(res, (list, tuple)): + for k in res: + self._persist(args, k) # 1-to-n replacement + return None + return self._persist(args, res) # normal & null replacement + + def _persist(self, args: 'GroupByCallbackArgs', obj: Any) \ + -> Optional[str]: ''' Update internal state. Return slugified string. ''' - alt = record.alt - slug = self.config.slugify(group) + if not isinstance(obj, (str, bool, int, float)) and obj is not None: + raise ValueError( + 'Unsupported groupby yield type for [{}]:' + ' {} (expected str, got {})'.format( + self.config.key, obj, type(obj).__name__)) + + if obj is None: + # if obj is not set, test if config.replace_none_key is set + slug = self.config.replace_none_key + obj = slug + else: + # if obj is set, apply config.key_map (convert int -> str) + slug = self.config.slugify(str(obj)) or None + # if neither custom mapping succeeded, do not process further + if not slug or obj is None: + return slug + # update internal object storage + alt = args.record.alt if slug not in self._state[alt]: src = GroupBySource(self._root_record[alt], slug) self._state[alt][slug] = src else: src = self._state[alt][slug] - src.append_child(record, group) + src.append_child(args.record, obj) # obj is used as "group" string # reverse reference - VGroups.of(record).add(key, src) + VGroups.of(args.record).add(args.key, src) return slug - def iter_sources(self, root: 'Record') -> Iterator[GroupBySource]: + def remember(self, record: 'Record') -> None: + self._rmmbr.append(record) + + def iter_sources(self) -> Iterator[GroupBySource]: ''' Prepare and yield GroupBySource elements. ''' + for x in self._rmmbr: + self.process(x) + del self._rmmbr for vobj_list in self._state.values(): for vobj in vobj_list.values(): yield vobj.finalize(self.config)