Refactor select with new maybe selector

This is a great refactor of the select functionality in the flake class.
This now uses the same parser as the nix code, but runs it in python for
nice stacktraces.

Also we now have a maybe selector which can be used by prepending the
selector with a ?

Tests have been expanded to make sure the code is more stable and easier
to understand

flake.lock

@@ -89,6 +89,21 @@
         "type": "github"
       }
     },
+    "nix-select": {
+      "locked": {
+        "lastModified": 1745005516,
+        "narHash": "sha256-IVaoOGDIvAa/8I0sdiiZuKptDldrkDWUNf/+ezIRhyc=",
+        "ref": "refs/heads/main",
+        "rev": "69d8bf596194c5c35a4e90dd02c52aa530caddf8",
+        "revCount": 40,
+        "type": "git",
+        "url": "https://git.clan.lol/clan/nix-select"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.clan.lol/clan/nix-select"
+      }
+    },
     "nixos-facter-modules": {
       "locked": {
         "lastModified": 1743671943,
@@ -123,6 +138,7 @@
         "disko": "disko",
         "flake-parts": "flake-parts",
         "nix-darwin": "nix-darwin",
+        "nix-select": "nix-select",
         "nixos-facter-modules": "nixos-facter-modules",
         "nixpkgs": "nixpkgs",
         "sops-nix": "sops-nix",

flake.nix

@@ -23,6 +23,8 @@
     treefmt-nix.url = "github:numtide/treefmt-nix";
     treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
 
+    nix-select.url = "git+https://git.clan.lol/clan/nix-select";
+
     data-mesher = {
       url = "git+https://git.clan.lol/clan/data-mesher";
       inputs = {

lib/default.nix

@@ -30,6 +30,6 @@ lib.fix (clanLib: {
   # Plain imports.
   values = import ./introspection { inherit lib; };
   jsonschema = import ./jsonschema { inherit lib; };
-  select = import select/default.nix;
+  select = self.inputs.nix-select.lib;
   facts = import ./facts.nix { inherit lib; };
 })

lib/flake-module.nix

@@ -11,7 +11,6 @@ rec {
     ./introspection/flake-module.nix
     ./inventory/flake-module.nix
     ./jsonschema/flake-module.nix
-    ./select/flake-module.nix
   ];
   flake.clanLib = import ./default.nix {
     inherit lib inputs self;

lib/select/default.nix

@@ -1,68 +0,0 @@
-let
-  recursiveSelect =
-    selectorIndex: selectorList: target:
-    let
-      selector = builtins.elemAt selectorList selectorIndex;
-    in
-
-    # selector is empty, we are done
-    if selectorIndex + 1 > builtins.length selectorList then
-      target
-
-    else if builtins.isList target then
-      # support bla.* for lists and recurse into all elements
-      if selector == "*" then
-        builtins.map (v: recursiveSelect (selectorIndex + 1) selectorList v) target
-      # support bla.3 for lists and recurse into the 4th element
-      else if (builtins.match "[[:digit:]]*" selector) == [ ] then
-        recursiveSelect (selectorIndex + 1) selectorList (
-          builtins.elemAt target (builtins.fromJSON selector)
-        )
-      else
-        throw "only * or a number is allowed in list selector"
-
-    else if builtins.isAttrs target then
-      # handle the case bla.x.*.z where x is an attrset and we recurse into all elements
-      if selector == "*" then
-        builtins.mapAttrs (_: v: recursiveSelect (selectorIndex + 1) selectorList v) target
-      # support bla.{x,y,z}.world where we get world from each of x, y and z
-      else if (builtins.match ''^\{([^}]*)}$'' selector) != null then
-        let
-          attrsAsList = (
-            builtins.filter (x: !builtins.isList x) (
-              builtins.split "," (builtins.head (builtins.match ''^\{([^}]*)}$'' selector))
-            )
-          );
-          dummyAttrSet = builtins.listToAttrs (
-            map (x: {
-              name = x;
-              value = null;
-            }) attrsAsList
-          );
-          filteredAttrs = builtins.intersectAttrs dummyAttrSet target;
-        in
-        builtins.mapAttrs (_: v: recursiveSelect (selectorIndex + 1) selectorList v) filteredAttrs
-      else
-        recursiveSelect (selectorIndex + 1) selectorList (builtins.getAttr selector target)
-    else
-      throw "Expected a list or an attrset";
-
-  parseSelector =
-    selector:
-    let
-      splitByQuote = x: builtins.filter (x: !builtins.isList x) (builtins.split ''"'' x);
-      splitByDot =
-        x:
-        builtins.filter (x: x != "") (
-          map (builtins.replaceStrings [ "." ] [ "" ]) (
-            builtins.filter (x: !builtins.isList x) (builtins.split ''\.'' x)
-          )
-        );
-      handleQuoted =
-        x: if x == [ ] then [ ] else [ (builtins.head x) ] ++ handleUnquoted (builtins.tail x);
-      handleUnquoted =
-        x: if x == [ ] then [ ] else splitByDot (builtins.head x) ++ handleQuoted (builtins.tail x);
-    in
-    handleUnquoted (splitByQuote selector);
-in
-selector: target: recursiveSelect 0 (parseSelector selector) target

lib/select/flake-module.nix

@@ -1,45 +0,0 @@
-{ self, inputs, ... }:
-let
-  inputOverrides = builtins.concatStringsSep " " (
-    builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs)
-  );
-in
-{
-  perSystem =
-    {
-      pkgs,
-      lib,
-      system,
-      ...
-    }:
-    {
-      # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests
-      legacyPackages.evalTests-select = import ./tests.nix {
-        inherit lib;
-        inherit (self) clanLib;
-      };
-
-      checks = {
-        lib-select-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
-          export HOME="$(realpath .)"
-          export NIX_ABORT_ON_WARN=1
-          nix-unit --eval-store "$HOME" \
-            --extra-experimental-features flakes \
-            --show-trace \
-            ${inputOverrides} \
-            --flake ${
-              self.filter {
-                include = [
-                  "flakeModules"
-                  "lib"
-                  "clanModules/flake-module.nix"
-                  "clanModules/borgbackup"
-                ];
-              }
-            }#legacyPackages.${system}.evalTests-select
-
-          touch $out
-        '';
-      };
-    };
-}

lib/select/tests.nix

@@ -1,10 +0,0 @@
-{ clanLib, ... }:
-let
-  inherit (clanLib) select;
-in
-{
-  test_simple_1 = {
-    expr = select "a" { a = 1; };
-    expected = 1;
-  };
-}

pkgs/clan-cli/clan_cli/flake.py

@@ -1,11 +1,11 @@
 import json
 import logging
-import re
+from dataclasses import asdict, dataclass, field
-from dataclasses import dataclass
+from enum import Enum
 from hashlib import sha1
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Any, cast
+from typing import Any
 
 from clan_cli.cmd import Log, RunOpts, run
 from clan_cli.dirs import user_cache_dir
@@ -21,42 +21,212 @@ from clan_cli.nix import (
 log = logging.getLogger(__name__)
 
 
-class AllSelector:
+class SetSelectorType(str, Enum):
-    pass
+    """
+    enum for the type of selector in a set.
+    For now this is either a string or a maybe selector.
+    """
+
+    STR = "str"
+    MAYBE = "maybe"
 
 
-Selector = str | int | AllSelector | set[int] | set[str]
+@dataclass
+class SetSelector:
+    """
+    This class represents a selector used in a set.
+    type: SetSelectorType = SetSelectorType.STR
+    value: str = ""
+
+    a set looks like this:
+    {key1,key2}
+    """
+
+    type: SetSelectorType = SetSelectorType.STR
+    value: str = ""
 
 
-def split_selector(selector: str) -> list[Selector]:
+class SelectorType(str, Enum):
+    """
+    enum for the type of a selector
+    this can be all, string, set or maybe
+    """
+
+    ALL = "all"
+    STR = "str"
+    SET = "set"
+    MAYBE = "maybe"
+
+
+@dataclass
+class Selector:
+    """
+    A class to represent a selector, which selects nix elements one level down.
+    consists of a SelectorType and a value.
+
+    if the type is all, no value is needed, since it selects all elements.
+    if the type is str, the value is a string, which is the key in a dict.
+    if the type is maybe the value is a string, which is the key in a dict.
+    if the type is set, the value is a list of SetSelector objects.
+    """
+
+    type: SelectorType = SelectorType.STR
+    value: str | list[SetSelector] | None = None
+
+    def as_dict(self) -> dict[str, Any]:
+        if self.type == SelectorType.SET:
+            assert isinstance(self.value, list)
+            return {
+                "type": self.type.value,
+                "value": [asdict(selector) for selector in self.value],
+            }
+        if self.type == SelectorType.ALL:
+            return {"type": self.type.value}
+        if self.type == SelectorType.STR:
+            assert isinstance(self.value, str)
+            return {"type": self.type.value, "value": self.value}
+        if self.type == SelectorType.MAYBE:
+            assert isinstance(self.value, str)
+            return {"type": self.type.value, "value": self.value}
+        msg = f"Invalid selector type: {self.type}"
+        raise ValueError(msg)
+
+
+def selectors_as_dict(selectors: list[Selector]) -> list[dict[str, Any]]:
+    return [selector.as_dict() for selector in selectors]
+
+
+def selectors_as_json(selectors: list[Selector]) -> str:
+    return json.dumps(selectors_as_dict(selectors))
+
+
+def parse_selector(selector: str) -> list[Selector]:
     """
     takes a string and returns a list of selectors.
 
     a selector can be:
     - a string, which is a key in a dict
     - an integer, which is an index in a list
-    - a set of strings, which are keys in a dict
+    - a set of strings or integers, which are keys in a dict or indices in a list.
-    - a set of integers, which are indices in a list
-    - a quoted string, which is a key in a dict
     - the string "*", which selects all elements in a list or dict
     """
-    pattern = r'"[^"]*"|[^.]+'
+    stack: list[str] = []
-    matches = re.findall(pattern, selector)
-
-    # Extract the matched groups (either quoted or unquoted parts)
     selectors: list[Selector] = []
-    for selector in matches:
+    acc_str: str = ""
-        if selector == "*":
+
-            selectors.append(AllSelector())
+    # only used by set for now
-        elif selector.isdigit():
+    submode = ""
-            selectors.append({int(selector)})
+    acc_selectors: list[SetSelector] = []
-        elif selector.startswith("{") and selector.endswith("}"):
+
-            sub_selectors = set(selector[1:-1].split(","))
+    for i in range(len(selector)):
-            selectors.append(sub_selectors)
+        c = selector[i]
-        elif selector.startswith('"') and selector.endswith('"'):
+        if stack == []:
-            selectors.append(selector[1:-1])
+            mode = "start"
         else:
-            selectors.append(selector)
+            mode = stack[-1]
+
+        if mode == "end":
+            if c == ".":
+                stack.pop()
+                if stack != []:
+                    msg = "expected empy stack, but got {stack}"
+                    raise ValueError(msg)
+            else:
+                msg = "expected ., but got {c}"
+                raise ValueError(msg)
+
+        elif mode == "start":
+            if c == "*":
+                stack.append("end")
+                selectors.append(Selector(type=SelectorType.ALL))
+            elif c == "?":
+                stack.append("maybe")
+            elif c == '"':
+                stack += ["str", "quote"]
+            elif c == "{":
+                stack.append("set")
+            elif c == ".":
+                selectors.append(Selector(type=SelectorType.STR, value=acc_str))
+            else:
+                stack.append("str")
+                acc_str += c
+
+        elif mode == "set":
+            if submode == "" and c == "?":
+                submode = "maybe"
+            elif c == "\\":
+                stack.append("escape")
+                if submode == "":
+                    submode = "str"
+            elif c == '"':
+                stack.append("quote")
+                if submode == "":
+                    submode = "str"
+            elif c == ",":
+                if submode == "maybe":
+                    set_select_type = SetSelectorType.MAYBE
+                else:
+                    set_select_type = SetSelectorType.STR
+                acc_selectors.append(SetSelector(type=set_select_type, value=acc_str))
+                submode = ""
+                acc_str = ""
+            elif c == "}":
+                if submode == "maybe":
+                    set_select_type = SetSelectorType.MAYBE
+                else:
+                    set_select_type = SetSelectorType.STR
+                acc_selectors.append(SetSelector(type=set_select_type, value=acc_str))
+                selectors.append(Selector(type=SelectorType.SET, value=acc_selectors))
+
+                submode = ""
+                acc_selectors = []
+
+                acc_str = ""
+                stack.pop()
+                stack.append("end")
+            else:
+                acc_str += c
+                if submode == "":
+                    submode = "str"
+
+        elif mode == "quote":
+            if c == '"':
+                stack.pop()
+            elif c == "\\":
+                stack.append("escape")
+            else:
+                acc_str += c
+
+        elif mode == "escape":
+            stack.pop()
+            acc_str += c
+
+        elif mode == "str" or mode == "maybe":
+            if c == ".":
+                stack.pop()
+                if mode == "maybe":
+                    select_type = SelectorType.MAYBE
+                else:
+                    select_type = SelectorType.STR
+                selectors.append(Selector(type=select_type, value=acc_str))
+                acc_str = ""
+            elif c == "\\":
+                stack.append("escape")
+            else:
+                acc_str += c
+
+    if stack != []:
+        if stack[-1] == "str" or stack[-1] == "maybe":
+            if stack[-1] == "maybe":
+                select_type = SelectorType.MAYBE
+            else:
+                select_type = SelectorType.STR
+            selectors.append(Selector(type=select_type, value=acc_str))
+        elif stack[-1] == "end":
+            pass
+        else:
+            msg = f"expected empty stack, but got {stack}"
+            raise ValueError(msg)
 
     return selectors
 
@@ -64,94 +234,18 @@ def split_selector(selector: str) -> list[Selector]:
 @dataclass
 class FlakeCacheEntry:
     """
-    a recursive structure to store the cache, with a value and a selector
+    a recursive structure to store the cache.
+    consists of a dict with the keys being the selectors and the values being FlakeCacheEntry objects.
+
+    is_list is used to check if the value is a list.
+    exists is used to check if the value exists, which can be false if it was selected via maybe.
+    fetched_all is used to check if we have all keys on the current level.
     """
 
-    def __init__(
+    value: str | float | dict[str, Any] | None = field(default_factory=dict)
-        self,
+    is_list: bool = False
-        value: str | float | dict[str, Any] | list[Any] | None,
+    exists: bool = True
-        selectors: list[Selector],
+    fetched_all: bool = False
-        is_out_path: bool = False,
-    ) -> None:
-        self.value: str | float | int | None | dict[str | int, FlakeCacheEntry]
-        self.selector: set[int] | set[str] | AllSelector
-        selector: Selector = AllSelector()
-
-        if selectors == []:
-            self.selector = AllSelector()
-        elif isinstance(selectors[0], set):
-            self.selector = selectors[0]
-            selector = selectors[0]
-        elif isinstance(selectors[0], int):
-            self.selector = {int(selectors[0])}
-            selector = int(selectors[0])
-        elif isinstance(selectors[0], str):
-            self.selector = {selectors[0]}
-            selector = selectors[0]
-        elif isinstance(selectors[0], AllSelector):
-            self.selector = AllSelector()
-
-        if is_out_path:
-            if selectors != []:
-                msg = "Cannot index outPath"
-                raise ValueError(msg)
-            if not isinstance(value, str):
-                msg = "outPath must be a string"
-                raise ValueError(msg)
-            self.value = value
-
-        elif isinstance(selector, str):
-            self.value = {selector: FlakeCacheEntry(value, selectors[1:])}
-
-        elif isinstance(value, dict):
-            if isinstance(self.selector, set):
-                if not all(isinstance(v, str) for v in self.selector):
-                    msg = "Cannot index dict with non-str set"
-                    raise ValueError(msg)
-            self.value = {}
-            for key, value_ in value.items():
-                if key == "outPath":
-                    self.value[key] = FlakeCacheEntry(
-                        value_, selectors[1:], is_out_path=True
-                    )
-                else:
-                    self.value[key] = FlakeCacheEntry(value_, selectors[1:])
-
-        elif isinstance(value, list):
-            if isinstance(selector, int):
-                if len(value) != 1:
-                    msg = "Cannot index list with int selector when value is not singleton"
-                    raise ValueError(msg)
-                self.value = {
-                    int(selector): FlakeCacheEntry(value[0], selectors[1:]),
-                }
-            if isinstance(selector, set):
-                if all(isinstance(v, int) for v in selector):
-                    self.value = {}
-                    for i, v in enumerate([selector]):
-                        assert isinstance(v, int)
-                        self.value[int(v)] = FlakeCacheEntry(value[i], selectors[1:])
-                else:
-                    msg = "Cannot index list with non-int set"
-                    raise ValueError(msg)
-            elif isinstance(self.selector, AllSelector):
-                self.value = {}
-                for i, v in enumerate(value):
-                    if isinstance(v, dict | list | str | float | int):
-                        self.value[i] = FlakeCacheEntry(v, selectors[1:])
-            else:
-                msg = f"expected integer selector or all for type list, but got {type(selector)}"
-                raise TypeError(msg)
-
-        elif isinstance(value, str) and value.startswith("/nix/store/"):
-            self.value = {}
-            self.selector = self.selector = {"outPath"}
-            self.value["outPath"] = FlakeCacheEntry(
-                value, selectors[1:], is_out_path=True
-            )
-
-        elif isinstance(value, (str | float | int | None)):
-            self.value = value
 
     def insert(
         self,
@@ -159,175 +253,236 @@ class FlakeCacheEntry:
         selectors: list[Selector],
     ) -> None:
         selector: Selector
+        # if we have no more selectors, it means we select all keys from now one and futher down
         if selectors == []:
-            selector = AllSelector()
+            selector = Selector(type=SelectorType.ALL)
         else:
             selector = selectors[0]
 
-        if isinstance(selector, str):
+        # first we find out if we have all subkeys already
-            if isinstance(self.value, dict):
-                if selector in self.value:
-                    self.value[selector].insert(value, selectors[1:])
-                else:
-                    self.value[selector] = FlakeCacheEntry(value, selectors[1:])
-                return
-            msg = f"Cannot insert {selector} into non dict value"
-            raise TypeError(msg)
 
-        if isinstance(selector, AllSelector):
+        if self.fetched_all:
-            self.selector = AllSelector()
+            pass
-        elif isinstance(self.selector, set) and isinstance(selector, set):
+        elif selector.type == SelectorType.ALL:
-            if all(isinstance(v, str) for v in self.selector) and all(
+            self.fetched_all = True
-                isinstance(v, str) for v in selector
-            ):
-                selector = cast(set[str], selector)
-                self.selector = cast(set[str], self.selector)
-                self.selector = self.selector.union(selector)
-            elif all(isinstance(v, int) for v in self.selector) and all(
-                isinstance(v, int) for v in selector
-            ):
-                selector = cast(set[int], selector)
-                self.selector = cast(set[int], self.selector)
-                self.selector = self.selector.union(selector)
-            else:
-                msg = "Cannot union set of different types"
-                raise ValueError(msg)
-        elif isinstance(self.selector, set) and isinstance(selector, int):
-            if all(isinstance(v, int) for v in self.selector):
-                self.selector = cast(set[int], self.selector)
-                self.selector.add(selector)
 
-        elif isinstance(self.selector, set) and isinstance(selector, str):
+        # if we have a string selector, that means we are usually on a dict or a list, since we cannot walk down scalar values
-            if all(isinstance(v, str) for v in self.selector):
+        # so we passthrough the value to the next level
-                self.selector = cast(set[str], self.selector)
+        if selector.type == SelectorType.STR:
-                self.selector.add(selector)
+            assert isinstance(selector.value, str)
+            assert isinstance(self.value, dict)
+            if selector.value not in self.value:
+                self.value[selector.value] = FlakeCacheEntry()
+            self.value[selector.value].insert(value, selectors[1:])
 
-        else:
+        # if we get a MAYBE, check if the selector is in the output, if not we create a entry with exists = False
-            msg = f"Cannot insert {selector} into {self.selector}"
+        # otherwise we just insert the value into the current dict
-            raise TypeError(msg)
+        # we can skip creating the non existing entry if we already fetched all keys
+        elif selector.type == SelectorType.MAYBE:
+            assert isinstance(self.value, dict)
+            assert isinstance(value, dict)
+            assert isinstance(selector.value, str)
+            if selector.value in value:
+                if selector.value not in self.value:
+                    self.value[selector.value] = FlakeCacheEntry()
+                self.value[selector.value].insert(value[selector.value], selectors[1:])
+            elif not self.fetched_all:
+                if selector.value not in self.value:
+                    self.value[selector.value] = FlakeCacheEntry()
+                self.value[selector.value].exists = False
 
-        if isinstance(self.value, dict) and isinstance(value, dict):
+        # insert a dict is pretty straight forward
+        elif isinstance(value, dict):
+            assert isinstance(self.value, dict)
             for key, value_ in value.items():
-                if key in self.value:
+                if key not in self.value:
-                    self.value[key].insert(value_, selectors[1:])
+                    self.value[key] = FlakeCacheEntry()
-                else:
+                self.value[key].insert(value_, selectors[1:])
-                    self.value[key] = FlakeCacheEntry(value_, selectors[1:])
 
-        elif isinstance(self.value, dict) and isinstance(value, list):
+        # to store a list we also use a dict, so we know which indices we have
-            if isinstance(selector, set):
+        elif isinstance(value, list):
-                if not all(isinstance(v, int) for v in selector):
+            self.is_list = True
-                    msg = "Cannot list with non-int set"
+            fetched_indices: list[str] = []
-                    raise ValueError(msg)
+            # if we are in a set, we take all the selectors
-                for realindex, requested_index in enumerate(selector):
+            if selector.type == SelectorType.SET:
-                    assert isinstance(requested_index, int)
+                assert isinstance(selector.value, list)
-                    if requested_index in self.value:
+                for subselector in selector.value:
-                        self.value[requested_index].insert(
+                    fetched_indices.append(subselector.value)
-                            value[realindex], selectors[1:]
+            # if it's just a str, that is the index
-                        )
+            elif selector.type == SelectorType.STR:
-            elif isinstance(selector, AllSelector):
+                assert isinstance(selector.value, str)
-                for index, v in enumerate(value):
+                fetched_indices = [selector.value]
-                    if index in self.value:
+            # otherwise we just take all the indices, which is the length of the list
-                        self.value[index].insert(v, selectors[1:])
+            elif selector.type == SelectorType.ALL:
-                    else:
+                fetched_indices = list(map(str, range(len(value))))
-                        self.value[index] = FlakeCacheEntry(v, selectors[1:])
+
-            elif isinstance(selector, int):
+            # insert is the same is insert a dict
-                if selector in self.value:
+            assert isinstance(self.value, dict)
-                    self.value[selector].insert(value[0], selectors[1:])
+            for i, requested_index in enumerate(fetched_indices):
-                else:
+                assert isinstance(requested_index, str)
-                    self.value[selector] = FlakeCacheEntry(value[0], selectors[1:])
+                if requested_index not in self.value:
+                    self.value[requested_index] = FlakeCacheEntry()
+                self.value[requested_index].insert(value[i], selectors[1:])
+
+        # strings need to be checked if they are store paths
+        # if they are, we store them as a dict with the outPath key
+        # this is to mirror nix behavior, where the outPath of an attrset is used if no further key is specified
         elif isinstance(value, str) and value.startswith("/nix/store/"):
-            self.value = {}
+            assert selectors == []
-            self.value["outPath"] = FlakeCacheEntry(
+            if value.startswith("/nix/store/"):
-                value, selectors[1:], is_out_path=True
+                self.value = {"outPath": FlakeCacheEntry(value)}
-            )
 
-        elif isinstance(value, (str | float | int)):
+        # if we have a normal scalar, we check if it conflicts with a maybe already store value
-            if self.value:
+        # since an empty attrset is the default value, we cannot check that, so we just set it to the value
-                if self.value != value:
+        elif isinstance(value, float | int | str) or value is None:
-                    msg = "value mismatch in cache, something is fishy"
+            assert selectors == []
-                    raise TypeError(msg)
+            if self.value == {}:
-
+                self.value = value
-        elif value is None:
+            elif self.value != value:
-            if self.value is not None:
+                msg = f"Cannot insert {value} into cache, already have {self.value}"
-                msg = "value mismatch in cache, something is fishy"
                 raise TypeError(msg)
 
-        else:
-            msg = f"Cannot insert value of type {type(value)} into cache"
-            raise TypeError(msg)
-
     def is_cached(self, selectors: list[Selector]) -> bool:
         selector: Selector
         if selectors == []:
-            selector = AllSelector()
+            return self.fetched_all
-        else:
+        selector = selectors[0]
-            selector = selectors[0]
 
+        # for store paths we have to check if they still exist, otherwise they have to be rebuild and are thus not cached
         if isinstance(self.value, str) and self.value.startswith("/nix/store/"):
             return Path(self.value).exists()
+
+        # if self.value is not dict but we request more selectors, we assume we are cached and an error will be thrown in the select function
         if isinstance(self.value, str | float | int | None):
-            return selectors == []
+            return True
-        if isinstance(selector, AllSelector):
+
-            if isinstance(self.selector, AllSelector):
+        # we just fetch all subkeys, so we need to check of we inserted all keys at this level before
+        if selector.type == SelectorType.ALL:
+            assert isinstance(self.value, dict)
+            if self.fetched_all:
                 result = all(
                     self.value[sel].is_cached(selectors[1:]) for sel in self.value
                 )
                 return result
-            # TODO: check if we already have all the keys anyway?
             return False
         if (
-            isinstance(selector, set)
+            selector.type == SelectorType.SET
-            and isinstance(self.selector, set)
+            and isinstance(selector.value, list)
             and isinstance(self.value, dict)
         ):
-            if not selector.issubset(self.selector):
+            for requested_selector in selector.value:
-                return False
+                val = requested_selector.value
+                if val not in self.value:
+                    # if we fetched all keys and we are not in the dict, we can assume we are cached
+                    return self.fetched_all
+                # if a key does not exist from a previous fetch, we can assume it is cached
+                if self.value[val].exists is False:
+                    return True
+                if not self.value[val].is_cached(selectors[1:]):
+                    return False
 
-            result = all(
+            return True
-                self.value[sel].is_cached(selectors[1:]) if sel in self.value else True
-                for sel in selector
-            )
 
-            return result
+        # string and maybe work the same for cache checking
-        if isinstance(selector, str | int) and isinstance(self.value, dict):
+        if (
-            if selector in self.value:
+            selector.type == SelectorType.STR or selector.type == SelectorType.MAYBE
-                result = self.value[selector].is_cached(selectors[1:])
+        ) and isinstance(self.value, dict):
-                return result
+            assert isinstance(selector.value, str)
-            return False
+            val = selector.value
+            if val not in self.value:
+                # if we fetched all keys and we are not in there, refetching won't help, so we can assume we are cached
+                return self.fetched_all
+            if self.value[val].exists is False:
+                return True
+            return self.value[val].is_cached(selectors[1:])
 
         return False
 
     def select(self, selectors: list[Selector]) -> Any:
         selector: Selector
         if selectors == []:
-            selector = AllSelector()
+            selector = Selector(type=SelectorType.ALL)
         else:
             selector = selectors[0]
 
+        # mirror nix behavior where we return outPath if no further selector is specified
         if selectors == [] and isinstance(self.value, dict) and "outPath" in self.value:
             return self.value["outPath"].value
 
-        if isinstance(self.value, str | float | int | None):
+        # if we are at the end of the selector chain, we return the value
+        if selectors == [] and isinstance(self.value, str | float | int | None):
             return self.value
+
+        # if we fetch a specific key, we return the recurse into that value in the dict
+        if selector.type == SelectorType.STR and isinstance(self.value, dict):
+            assert isinstance(selector.value, str)
+            return self.value[selector.value].select(selectors[1:])
+
+        # if we are a MAYBE selector, we check if the key exists in the dict
+        if selector.type == SelectorType.MAYBE and isinstance(self.value, dict):
+            assert isinstance(selector.value, str)
+            if selector.value in self.value:
+                if self.value[selector.value].exists:
+                    return {
+                        selector.value: self.value[selector.value].select(selectors[1:])
+                    }
+                return {}
+            if self.fetched_all:
+                return {}
+
+        # otherwise we return a list or a dict
         if isinstance(self.value, dict):
-            if isinstance(selector, AllSelector):
+            keys_to_select: list[str] = []
-                return {k: v.select(selectors[1:]) for k, v in self.value.items()}
+            # if we want to select all keys, we take all existing sub elements
-            if isinstance(selector, set):
+            if selector.type == SelectorType.ALL:
-                return {
+                for key in self.value:
-                    k: v.select(selectors[1:])
+                    if self.value[key].exists:
-                    for k, v in self.value.items()
+                        keys_to_select.append(key)
-                    if k in selector
+
-                }
+            # if we want to select a set of keys, we take the keys from the selector
-            if isinstance(selector, str | int):
+            if selector.type == SelectorType.SET:
-                return self.value[selector].select(selectors[1:])
+                assert isinstance(selector.value, list)
-        msg = f"Cannot select {selector} from type {type(self.value)}"
+                for subselector in selector.value:
-        raise TypeError(msg)
+                    # make sure the keys actually exist if we have a maybe selector
+                    if subselector.type == SetSelectorType.MAYBE:
+                        if (
+                            subselector.value in self.value
+                            and self.value[subselector.value].exists
+                        ):
+                            keys_to_select.append(subselector.value)
+                    else:
+                        keys_to_select.append(subselector.value)
+
+            # if we are a list, return a list
+            if self.is_list:
+                result = []
+                for index in keys_to_select:
+                    result.append(self.value[index].select(selectors[1:]))
+                    return result
+
+            # otherwise return a dict
+            return {k: self.value[k].select(selectors[1:]) for k in keys_to_select}
+
+        # return a KeyError if we cannot fetch the key
+        str_selector: str
+        if selector.type == SelectorType.ALL:
+            str_selector = "*"
+        elif selector.type == SelectorType.SET:
+            subselectors: list[str] = []
+            assert isinstance(selector.value, list)
+            for subselector in selector.value:
+                subselectors.append(subselector.value)
+            str_selector = "{" + ",".join(subselectors) + "}"
+        else:
+            assert isinstance(selector.value, str)
+            str_selector = selector.value
+
+        raise KeyError(str_selector)
 
     def __getitem__(self, name: str) -> "FlakeCacheEntry":
         if isinstance(self.value, dict):
             return self.value[name]
-        msg = f"value is a {type(self.value)}, so cannot subscribe"
+        raise KeyError(name)
-        raise TypeError(msg)
 
     def as_json(self) -> dict[str, Any]:
         json_data: Any = {}
@@ -338,21 +493,13 @@ class FlakeCacheEntry:
         else:  # == str | float | None
             json_data["value"] = self.value
 
-        if isinstance(self.selector, AllSelector):
+        json_data["is_list"] = self.is_list
-            json_data["selector"] = "all-selector"
+        json_data["exists"] = self.exists
-        else:  # == set[int] | set[str]
+        json_data["fetched_all"] = self.fetched_all
-            json_data["selector"] = list(self.selector)
         return json_data
 
     @staticmethod
     def from_json(json_data: dict[str, Any]) -> "FlakeCacheEntry":
-        raw_selector = json_data.get("selector")
-        if raw_selector == "all-selector":
-            selector: Any = AllSelector()
-        else:  # == set[int] | set[str]
-            assert isinstance(raw_selector, list)
-            selector = set(raw_selector)
-
         raw_value = json_data.get("value")
         if isinstance(raw_value, dict):
             value: Any = {}
@@ -361,9 +508,13 @@ class FlakeCacheEntry:
         else:  # == str | float | None
             value = raw_value
 
-        entry = FlakeCacheEntry(None, [], is_out_path=False)
+        is_list = json_data.get("is_list", False)
-        entry.selector = selector
+        exists = json_data.get("exists", True)
-        entry.value = value
+        fetched_all = json_data.get("fetched_all", False)
+
+        entry = FlakeCacheEntry(
+            value=value, is_list=is_list, exists=exists, fetched_all=fetched_all
+        )
         return entry
 
     def __repr__(self) -> str:
@@ -379,22 +530,22 @@ class FlakeCache:
     """
 
     def __init__(self) -> None:
-        self.cache: FlakeCacheEntry = FlakeCacheEntry({}, [])
+        self.cache: FlakeCacheEntry = FlakeCacheEntry()
 
     def insert(self, data: dict[str, Any], selector_str: str) -> None:
         if selector_str:
-            selectors = split_selector(selector_str)
+            selectors = parse_selector(selector_str)
         else:
             selectors = []
 
         self.cache.insert(data, selectors)
 
     def select(self, selector_str: str) -> Any:
-        selectors = split_selector(selector_str)
+        selectors = parse_selector(selector_str)
         return self.cache.select(selectors)
 
     def is_cached(self, selector_str: str) -> bool:
-        selectors = split_selector(selector_str)
+        selectors = parse_selector(selector_str)
         return self.cache.is_cached(selectors)
 
     def save_to_file(self, path: Path) -> None:
@@ -544,13 +695,17 @@ class Flake:
         if nix_options is None:
             nix_options = []
 
+        str_selectors: list[str] = []
+        for selector in selectors:
+            str_selectors.append(selectors_as_json(parse_selector(selector)))
+
         config = nix_config()
         nix_code = f"""
             let
               flake = builtins.getFlake("path:{self.store_path}?narHash={self.hash}");
             in
               flake.inputs.nixpkgs.legacyPackages.{config["system"]}.writeText "clan-flake-select" (
-                builtins.toJSON [ {" ".join([f"(flake.clanInternals.clanLib.select ''{attr}'' flake)" for attr in selectors])} ]
+                builtins.toJSON [ {" ".join([f"(flake.clanInternals.clanLib.select.applySelectors (builtins.fromJSON ''{attr}'') flake)" for attr in str_selectors])} ]
               )
         """
         if tmp_store := nix_test_store():

pkgs/clan-cli/clan_cli/tests/test_flake_caching.py

@@ -1,35 +1,282 @@
 import logging
 
 import pytest
-from clan_cli.flake import Flake, FlakeCache, FlakeCacheEntry
+from clan_cli.flake import (
+    Flake,
+    FlakeCache,
+    FlakeCacheEntry,
+    parse_selector,
+    selectors_as_dict,
+)
 from clan_cli.tests.fixtures_flakes import ClanFlake
 
 log = logging.getLogger(__name__)
 
 
+def test_parse_selector() -> None:
+    selectors = parse_selector("x")
+    assert selectors_as_dict(selectors) == [
+        {"type": "str", "value": "x"},
+    ]
+    selectors = parse_selector("?x")
+    assert selectors_as_dict(selectors) == [
+        {"type": "maybe", "value": "x"},
+    ]
+    selectors = parse_selector('"x"')
+    assert selectors_as_dict(selectors) == [
+        {"type": "str", "value": "x"},
+    ]
+    selectors = parse_selector("*")
+    assert selectors_as_dict(selectors) == [
+        {"type": "all"},
+    ]
+    selectors = parse_selector("{x}")
+    assert selectors_as_dict(selectors) == [
+        {
+            "type": "set",
+            "value": [
+                {"type": "str", "value": "x"},
+            ],
+        },
+    ]
+    selectors = parse_selector("{x}.y")
+    assert selectors_as_dict(selectors) == [
+        {
+            "type": "set",
+            "value": [
+                {"type": "str", "value": "x"},
+            ],
+        },
+        {"type": "str", "value": "y"},
+    ]
+    selectors = parse_selector("x.y.z")
+    assert selectors_as_dict(selectors) == [
+        {"type": "str", "value": "x"},
+        {"type": "str", "value": "y"},
+        {"type": "str", "value": "z"},
+    ]
+    selectors = parse_selector("x.*")
+    assert selectors_as_dict(selectors) == [
+        {"type": "str", "value": "x"},
+        {"type": "all"},
+    ]
+    selectors = parse_selector("*.x")
+    assert selectors_as_dict(selectors) == [
+        {"type": "all"},
+        {"type": "str", "value": "x"},
+    ]
+    selectors = parse_selector("x.*.z")
+    assert selectors_as_dict(selectors) == [
+        {"type": "str", "value": "x"},
+        {"type": "all"},
+        {"type": "str", "value": "z"},
+    ]
+    selectors = parse_selector("x.{y,z}")
+    assert selectors_as_dict(selectors) == [
+        {"type": "str", "value": "x"},
+        {
+            "type": "set",
+            "value": [
+                {"type": "str", "value": "y"},
+                {"type": "str", "value": "z"},
+            ],
+        },
+    ]
+    selectors = parse_selector("x.?zzz.{y,?z,x,*}")
+    assert selectors_as_dict(selectors) == [
+        {"type": "str", "value": "x"},
+        {"type": "maybe", "value": "zzz"},
+        {
+            "type": "set",
+            "value": [
+                {"type": "str", "value": "y"},
+                {"type": "maybe", "value": "z"},
+                {"type": "str", "value": "x"},
+                {"type": "str", "value": "*"},
+            ],
+        },
+    ]
+
+    selectors = parse_selector('x."?zzz".?zzz\\.asd..{y,\\?z,"x,",*}')
+    assert selectors_as_dict(selectors) == [
+        {"type": "str", "value": "x"},
+        {"type": "str", "value": "?zzz"},
+        {"type": "maybe", "value": "zzz.asd"},
+        {"type": "str", "value": ""},
+        {
+            "type": "set",
+            "value": [
+                {"type": "str", "value": "y"},
+                {"type": "str", "value": "?z"},
+                {"type": "str", "value": "x,"},
+                {"type": "str", "value": "*"},
+            ],
+        },
+    ]
+
+
+def test_insert_and_iscached() -> None:
+    test_cache = FlakeCacheEntry()
+    selectors = parse_selector("x.y.z")
+    test_cache.insert("x", selectors)
+    assert test_cache["x"]["y"]["z"].value == "x"
+    assert test_cache.is_cached(selectors)
+    assert not test_cache.is_cached(parse_selector("x.y"))
+    assert test_cache.is_cached(parse_selector("x.y.z.1"))
+    assert not test_cache.is_cached(parse_selector("x.*.z"))
+    assert test_cache.is_cached(parse_selector("x.{y}.z"))
+    assert test_cache.is_cached(parse_selector("x.?y.z"))
+    assert not test_cache.is_cached(parse_selector("x.?z.z"))
+
+    test_cache = FlakeCacheEntry()
+    selectors = parse_selector("x.*.z")
+    test_cache.insert({"y": "x"}, selectors)
+    assert test_cache["x"]["y"]["z"].value == "x"
+    assert test_cache.is_cached(selectors)
+    assert not test_cache.is_cached(parse_selector("x.y"))
+    assert not test_cache.is_cached(parse_selector("x.y.x"))
+    assert test_cache.is_cached(parse_selector("x.y.z.1"))
+    assert test_cache.is_cached(parse_selector("x.{y}.z"))
+    assert test_cache.is_cached(parse_selector("x.{y,z}.z"))
+    assert test_cache.is_cached(parse_selector("x.{y,?z}.z"))
+    assert test_cache.is_cached(parse_selector("x.?y.z"))
+    assert test_cache.is_cached(parse_selector("x.?z.z"))
+
+    test_cache = FlakeCacheEntry()
+    selectors = parse_selector("x.{y}.z")
+    test_cache.insert({"y": "x"}, selectors)
+    assert test_cache["x"]["y"]["z"].value == "x"
+    assert test_cache.is_cached(selectors)
+    assert not test_cache.is_cached(parse_selector("x.y"))
+    assert test_cache.is_cached(parse_selector("x.y.z.1"))
+    assert not test_cache.is_cached(parse_selector("x.*.z"))
+    assert test_cache.is_cached(parse_selector("x.{y}.z"))
+    assert test_cache.is_cached(parse_selector("x.?y.z"))
+    assert not test_cache.is_cached(parse_selector("x.?z.z"))
+
+    test_cache = FlakeCacheEntry()
+    selectors = parse_selector("x.?y.z")
+    test_cache.insert({"y": "x"}, selectors)
+    assert test_cache["x"]["y"]["z"].value == "x"
+    assert test_cache.is_cached(selectors)
+    assert not test_cache.is_cached(parse_selector("x.y"))
+    assert test_cache.is_cached(parse_selector("x.y.z.1"))
+    assert not test_cache.is_cached(parse_selector("x.*.z"))
+    assert test_cache.is_cached(parse_selector("x.{y}.z"))
+    assert test_cache.is_cached(parse_selector("x.?y.z"))
+    assert not test_cache.is_cached(parse_selector("x.?z.z"))
+
+    test_cache = FlakeCacheEntry()
+    selectors = parse_selector("x.?y.z")
+    test_cache.insert({}, selectors)
+    assert test_cache["x"]["y"].exists is False
+    assert test_cache.is_cached(selectors)
+    assert test_cache.is_cached(parse_selector("x.y"))
+    assert test_cache.is_cached(parse_selector("x.y.z.1"))
+    assert test_cache.is_cached(parse_selector("x.?y.z.1"))
+    assert not test_cache.is_cached(parse_selector("x.*.z"))
+    assert test_cache.is_cached(parse_selector("x.{y}.z"))
+    assert test_cache.is_cached(parse_selector("x.?y.abc"))
+    assert not test_cache.is_cached(parse_selector("x.?z.z"))
+
+    test_cache = FlakeCacheEntry()
+    selectors = parse_selector("x.{y,z}.z")
+    test_cache.insert({"y": 1, "z": 2}, selectors)
+    assert test_cache["x"]["y"]["z"].value == 1
+    assert test_cache["x"]["z"]["z"].value == 2
+    assert test_cache.is_cached(selectors)
+    assert not test_cache.is_cached(parse_selector("x.y"))
+    assert test_cache.is_cached(parse_selector("x.y.z.1"))
+    assert not test_cache.is_cached(parse_selector("x.*.z"))
+    assert test_cache.is_cached(parse_selector("x.{y}.z"))
+    assert not test_cache.is_cached(parse_selector("x.?y.abc"))
+    assert test_cache.is_cached(parse_selector("x.?z.z"))
+
+    test_cache = FlakeCacheEntry()
+    selectors = parse_selector("x.y")
+    test_cache.insert(1, selectors)
+    selectors = parse_selector("x.z")
+    test_cache.insert(2, selectors)
+    assert test_cache["x"]["y"].value == 1
+    assert test_cache["x"]["z"].value == 2
+    assert test_cache.is_cached(parse_selector("x.y"))
+    assert test_cache.is_cached(parse_selector("x.y.z.1"))
+    assert not test_cache.is_cached(parse_selector("x.*.z"))
+    assert test_cache.is_cached(parse_selector("x.{y}.z"))
+    assert test_cache.is_cached(parse_selector("x.?y.abc"))
+    assert test_cache.is_cached(parse_selector("x.?z.z"))
+    assert not test_cache.is_cached(parse_selector("x.?x.z"))
+
+    test_cache = FlakeCacheEntry()
+    selectors = parse_selector("x.y.z")
+    test_cache.insert({"a": {"b": {"c": 1}}}, selectors)
+    assert test_cache.is_cached(selectors)
+    assert test_cache.is_cached(parse_selector("x.y.z.a.b.c"))
+    assert test_cache.is_cached(parse_selector("x.y.z.a.b"))
+    assert test_cache.is_cached(parse_selector("x.y.z.a"))
+    assert test_cache.is_cached(parse_selector("x.y.z"))
+    assert not test_cache.is_cached(parse_selector("x.y"))
+    assert not test_cache.is_cached(parse_selector("x"))
+    assert test_cache.is_cached(parse_selector("x.y.z.xxx"))
+
+    test_cache = FlakeCacheEntry()
+    selectors = parse_selector("x.y")
+    test_cache.insert(1, selectors)
+    with pytest.raises(TypeError):
+        test_cache.insert(2, selectors)
+    assert test_cache["x"]["y"].value == 1
+
+
 def test_select() -> None:
+    test_cache = FlakeCacheEntry()
+
+    test_cache.insert("bla", parse_selector("a.b.c"))
+    assert test_cache.select(parse_selector("a.b.c")) == "bla"
+    assert test_cache.select(parse_selector("a.b")) == {"c": "bla"}
+    assert test_cache.select(parse_selector("a")) == {"b": {"c": "bla"}}
+    assert test_cache.select(parse_selector("a.b.?c")) == {"c": "bla"}
+    assert test_cache.select(parse_selector("a.?b.?c")) == {"b": {"c": "bla"}}
+    assert test_cache.select(parse_selector("a.?c")) == {}
+    assert test_cache.select(parse_selector("a.?x.c")) == {}
+    assert test_cache.select(parse_selector("a.*")) == {"b": {"c": "bla"}}
+    assert test_cache.select(parse_selector("a.*.*")) == {"b": {"c": "bla"}}
+    assert test_cache.select(parse_selector("a.*.c")) == {"b": "bla"}
+    assert test_cache.select(parse_selector("a.b.*")) == {"c": "bla"}
+    assert test_cache.select(parse_selector("a.{b}.c")) == {"b": "bla"}
+    assert test_cache.select(parse_selector("a.{b}.{c}")) == {"b": {"c": "bla"}}
+    assert test_cache.select(parse_selector("a.b.{c}")) == {"c": "bla"}
+    assert test_cache.select(parse_selector("a.{?b}.c")) == {"b": "bla"}
+    assert test_cache.select(parse_selector("a.{?b,?x}.c")) == {"b": "bla"}
+    with pytest.raises(KeyError):
+        test_cache.select(parse_selector("a.b.x"))
+    with pytest.raises(KeyError):
+        test_cache.select(parse_selector("a.b.c.x"))
+    with pytest.raises(KeyError):
+        test_cache.select(parse_selector("a.{b,x}.c"))
+
     testdict = {"x": {"y": [123, 345, 456], "z": "bla"}}
-    test_cache = FlakeCacheEntry(testdict, [])
+    test_cache.insert(testdict, parse_selector("testdict"))
-    assert test_cache["x"]["z"].value == "bla"
+    assert test_cache["testdict"]["x"]["z"].value == "bla"
-    assert test_cache.is_cached(["x", "z"])
+    selectors = parse_selector("testdict.x.z")
-    assert not test_cache.is_cached(["x", "y", "z"])
+    assert test_cache.select(selectors) == "bla"
-    assert test_cache.select(["x", "y", 0]) == 123
+    selectors = parse_selector("testdict.x.z.z")
-    assert not test_cache.is_cached(["x", "z", 1])
+    with pytest.raises(KeyError):
-
+        test_cache.select(selectors)
-
+    selectors = parse_selector("testdict.x.y.0")
-def test_insert() -> None:
+    assert test_cache.select(selectors) == 123
-    test_cache = FlakeCacheEntry({}, [])
+    selectors = parse_selector("testdict.x.z.1")
-    # Inserting the same thing twice should succeed
+    with pytest.raises(KeyError):
-    test_cache.insert(None, ["nix"])
+        test_cache.select(selectors)
-    test_cache.insert(None, ["nix"])
-    assert test_cache.select(["nix"]) is None
 
 
 def test_out_path() -> None:
     testdict = {"x": {"y": [123, 345, 456], "z": "/nix/store/bla"}}
-    test_cache = FlakeCacheEntry(testdict, [])
+    test_cache = FlakeCacheEntry()
-    assert test_cache.select(["x", "z"]) == "/nix/store/bla"
+    test_cache.insert(testdict, [])
-    assert test_cache.select(["x", "z", "outPath"]) == "/nix/store/bla"
+    selectors = parse_selector("x.z")
+    assert test_cache.select(selectors) == "/nix/store/bla"
+    selectors = parse_selector("x.z.outPath")
+    assert test_cache.select(selectors) == "/nix/store/bla"
 
 
 @pytest.mark.with_core
@@ -85,10 +332,10 @@ def test_conditional_all_selector(flake: ClanFlake) -> None:
     assert isinstance(flake1._cache, FlakeCache)  # noqa: SLF001
     assert isinstance(flake2._cache, FlakeCache)  # noqa: SLF001
     log.info("First select")
-    res1 = flake1.select("inputs.*.{clan,missing}")
+    res1 = flake1.select("inputs.*.{?clan,?missing}")
 
     log.info("Second (cached) select")
-    res2 = flake1.select("inputs.*.{clan,missing}")
+    res2 = flake1.select("inputs.*.{?clan,?missing}")
 
     assert res1 == res2
     assert res1["clan-core"].get("clan") is not None