Merge pull request 'docs: docs-integrity enable UTF-8 support' (#5413) from kenji/ke-html-proofer into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/5413

devFlake/flake.lock

@@ -3,10 +3,10 @@
     "clan-core-for-checks": {
       "flake": false,
       "locked": {
-        "lastModified": 1759727242,
-        "narHash": "sha256-15Q9eXbfsLmzIbYWasZ3Nuqafnc5o9al9RmGuBGVK74=",
+        "lastModified": 1759795610,
+        "narHash": "sha256-YFOK+aoJjWLfMHj2spvrQIe0ufIsv6P8o44NqoFPwp0=",
         "ref": "main",
-        "rev": "c737271585ff3df308feab22c09967fce8f278d3",
+        "rev": "0de79962eacfe6f09d7aabca2a7305deef4fde0c",
         "shallow": true,
         "type": "git",
         "url": "https://git.clan.lol/clan/clan-core"
@@ -105,11 +105,11 @@
     },
     "nixpkgs-dev": {
       "locked": {
-        "lastModified": 1759670943,
-        "narHash": "sha256-JBjTDfwzAwtd8+5X/Weg27WE/3hVYOP3uggP2JPaQVQ=",
+        "lastModified": 1759794031,
+        "narHash": "sha256-Zruni/00BlDHSWVJf3mb0o+OHnxIvJNuXkPloY9c+PU=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "21980a9c20f34648121f60bda15f419fa568db21",
+        "rev": "09c221b2f0726da85b124efb60a1d123971dfa08",
         "type": "github"
       },
       "original": {

docs/nix/flake-module.nix

@@ -121,6 +121,7 @@
         pkgs.runCommand "docs-integrity"
           {
             nativeBuildInputs = [ pkgs.html-proofer ];
+            LANG = "C.UTF-8";
           }
           ''
             # External links should be avoided in the docs, because they often break

docs/site/guides/nixpkgs-flake-input/index.md

@@ -5,11 +5,11 @@
 ## Option 1: Follow `clan-core`
 
 - **Pros**:
-  - Recommended for most users.
-  - Verified by our CI and widely used by others.
+    - Recommended for most users.
+    - Verified by our CI and widely used by others.
 - **Cons**:
-  - Coupled to version bumps in `clan-core`.
-  - Upstream features and packages may take longer to land.
+    - Coupled to version bumps in `clan-core`.
+    - Upstream features and packages may take longer to land.
 
 Example:
 
@@ -24,10 +24,10 @@ inputs = {
 ## Option 2: Use Your Own `nixpkgs` Version
 
 - **Pros**:
-  - Faster access to new upstream features and packages.
+    - Faster access to new upstream features and packages.
 - **Cons**:
-  - Recommended for advanced users.
-  - Not covered by our CI — you’re on the frontier.
+    - Recommended for advanced users.
+    - Not covered by our CI — you’re on the frontier.
 
 Example:
 

pkgs/clan-cli/clan_cli/secrets/sops.py

@@ -355,7 +355,10 @@ def get_public_age_key_from_private_key(privkey: str) -> str:
     cmd = nix_shell(["age"], ["age-keygen", "-y"])
 
     error_msg = "Failed to get public key for age private key. Is the key malformed?"
-    res = run(cmd, RunOpts(input=privkey.encode(), error_msg=error_msg))
+    res = run(
+        cmd,
+        RunOpts(input=privkey.encode(), error_msg=error_msg, sensitive_input=True),
+    )
     return res.stdout.rstrip(os.linesep).rstrip()
 
 

pkgs/clan-cli/clan_lib/cmd/__init__.py

@@ -74,9 +74,6 @@ def handle_io(
     )  # wlist is a list of file descriptors to be monitored for write events
     stdout_buf = b""
     stderr_buf = b""
-    # Buffers for incomplete lines (no trailing newline yet)
-    stdout_line_buf = ""
-    stderr_line_buf = ""
     start = time.time()
 
     # Function to handle file descriptors
@@ -88,40 +85,6 @@ def handle_io(
             rlist.remove(fd)
         return b""
 
-    # Function to process output with proper carriage return handling
-    def process_output(
-        chunk: bytes, line_buf: str, extra: dict[str, str], cmdlog_func: Any
-    ) -> str:
-        """Process output chunk, handling carriage returns properly.
-        Returns the updated line buffer (incomplete lines).
-        """
-        if not chunk:
-            return line_buf
-
-        # Decode the chunk and append to line buffer
-        decoded = chunk.decode("utf-8", "replace")
-        line_buf += decoded
-
-        # Split by newlines to get complete lines
-        lines = line_buf.split("\n")
-
-        # The last element might be an incomplete line
-        line_buf = lines[-1]
-        complete_lines = lines[:-1]
-
-        # Process each complete line
-        for line in complete_lines:
-            if "\r" in line:
-                # Handle carriage return: only keep the last segment after final \r
-                # This is what would be visible on a terminal
-                visible_line = line.split("\r")[-1]
-                if visible_line:  # Only log non-empty lines
-                    cmdlog_func(visible_line, extra=extra)
-            elif line:  # Only log non-empty lines
-                cmdlog_func(line, extra=extra)
-
-        return line_buf
-
     # Extra information passed to the logger
     stdout_extra = {}
     stderr_extra = {}
@@ -163,9 +126,9 @@ def handle_io(
 
         # If Log.STDOUT is set, log the stdout output
         if ret and log in [Log.STDOUT, Log.BOTH]:
-            stdout_line_buf = process_output(
-                ret, stdout_line_buf, stdout_extra, cmdlog.info
-            )
+            lines = ret.decode("utf-8", "replace").rstrip("\n").rstrip().split("\n")
+            for line in lines:
+                cmdlog.info(line, extra=stdout_extra)
 
         # If stdout file is set, stream the stdout output
         if ret and stdout:
@@ -180,9 +143,9 @@ def handle_io(
 
         # If Log.STDERR is set, log the stderr output
         if ret and log in [Log.STDERR, Log.BOTH]:
-            stderr_line_buf = process_output(
-                ret, stderr_line_buf, stderr_extra, cmdlog.info
-            )
+            lines = ret.decode("utf-8", "replace").rstrip("\n").rstrip().split("\n")
+            for line in lines:
+                cmdlog.info(line, extra=stderr_extra)
 
         # If stderr file is set, stream the stderr output
         if ret and stderr:
@@ -210,24 +173,6 @@ def handle_io(
                     process.stdin.close()
             else:
                 wlist.remove(process.stdin)
-
-    # Flush any remaining buffered lines at the end
-    if stdout_line_buf and log in [Log.STDOUT, Log.BOTH]:
-        if "\r" in stdout_line_buf:
-            visible_line = stdout_line_buf.split("\r")[-1]
-            if visible_line:
-                cmdlog.info(visible_line, extra=stdout_extra)
-        elif stdout_line_buf:
-            cmdlog.info(stdout_line_buf, extra=stdout_extra)
-
-    if stderr_line_buf and log in [Log.STDERR, Log.BOTH]:
-        if "\r" in stderr_line_buf:
-            visible_line = stderr_line_buf.split("\r")[-1]
-            if visible_line:
-                cmdlog.info(visible_line, extra=stderr_extra)
-        elif stderr_line_buf:
-            cmdlog.info(stderr_line_buf, extra=stderr_extra)
-
     return stdout_buf.decode("utf-8", "replace"), stderr_buf.decode("utf-8", "replace")
 
 
@@ -349,6 +294,8 @@ class RunOpts:
     # This is needed for GUI applications
     graphical_perm: bool = False
     trace: bool = True
+    # Mark input as sensitive to prevent it from being logged (e.g., private keys, passwords)
+    sensitive_input: bool = False
 
 
 def cmd_with_root(cmd: list[str], graphical: bool = False) -> list[str]:
@@ -404,7 +351,10 @@ def run(
 
     if cmdlog.isEnabledFor(logging.DEBUG) and options.trace:
         if options.input and isinstance(options.input, bytes):
-            if any(
+            # Always redact sensitive input (e.g., private keys, passwords)
+            if options.sensitive_input:
+                filtered_input = "<<REDACTED>>"
+            elif any(
                 not ch.isprintable() for ch in options.input.decode("ascii", "replace")
             ):
                 filtered_input = "<<binary_blob>>"