From edf0f9106ab3564f82b76debb44220baea19e67e Mon Sep 17 00:00:00 2001 From: Qubasa Date: Wed, 7 Aug 2024 15:49:39 +0200 Subject: [PATCH 1/4] clan-app: Initial move to process based api calling instead of thread based --- pkgs/clan-app/clan_app/api/__init__.py | 4 + pkgs/clan-app/clan_app/api/file.py | 12 ++ pkgs/clan-app/clan_app/components/executor.py | 127 ++++++++++++++++++ pkgs/clan-app/pyproject.toml | 4 - 4 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 pkgs/clan-app/clan_app/components/executor.py diff --git a/pkgs/clan-app/clan_app/api/__init__.py b/pkgs/clan-app/clan_app/api/__init__.py index c577dc467..73d424ac7 100644 --- a/pkgs/clan-app/clan_app/api/__init__.py +++ b/pkgs/clan-app/clan_app/api/__init__.py @@ -60,6 +60,10 @@ class ImplFunc(GObject.Object, Generic[P, B]): return result +# TODO: Reimplement this such that it uses a multiprocessing.Array of type ctypes.c_char +# all fn arguments are serialized to json and passed to the new process over the Array +# the new process deserializes the json and calls the function +# the result is serialized to json and passed back to the main process over another Array class MethodExecutor(threading.Thread): def __init__( self, function: Callable[..., Any], *args: Any, **kwargs: dict[str, Any] diff --git a/pkgs/clan-app/clan_app/api/file.py b/pkgs/clan-app/clan_app/api/file.py index 785e52eaa..c3235f926 100644 --- a/pkgs/clan-app/clan_app/api/file.py +++ b/pkgs/clan-app/clan_app/api/file.py @@ -54,6 +54,10 @@ class open_file( op_key=op_key, data=selected_paths, status="success" ) ) + else: + self.returns( + SuccessDataClass(op_key=op_key, data=None, status="success") + ) except Exception as e: print(f"Error getting selected files: {e}") @@ -67,6 +71,10 @@ class open_file( op_key=op_key, data=selected_path, status="success" ) ) + else: + self.returns( + SuccessDataClass(op_key=op_key, data=None, status="success") + ) except Exception as e: print(f"Error getting selected directory: {e}") @@ -80,6 +88,10 @@ class open_file( op_key=op_key, data=selected_path, status="success" ) ) + else: + self.returns( + SuccessDataClass(op_key=op_key, data=None, status="success") + ) except Exception as e: print(f"Error getting selected file: {e}") diff --git a/pkgs/clan-app/clan_app/components/executor.py b/pkgs/clan-app/clan_app/components/executor.py new file mode 100644 index 000000000..b97013c50 --- /dev/null +++ b/pkgs/clan-app/clan_app/components/executor.py @@ -0,0 +1,127 @@ +import dataclasses +import logging +import multiprocessing as mp +import os +import signal +import sys +import traceback +from collections.abc import Callable +from pathlib import Path +from typing import Any + +log = logging.getLogger(__name__) + + +# Kill the new process and all its children by sending a SIGTERM signal to the process group +def _kill_group(proc: mp.Process) -> None: + pid = proc.pid + if proc.is_alive() and pid: + os.killpg(pid, signal.SIGTERM) + else: + log.warning(f"Process '{proc.name}' with pid '{pid}' is already dead") + + +@dataclasses.dataclass(frozen=True) +class MPProcess: + name: str + proc: mp.Process + out_file: Path + + # Kill the new process and all its children by sending a SIGTERM signal to the process group + def kill_group(self) -> None: + _kill_group(proc=self.proc) + + +def _set_proc_name(name: str) -> None: + if sys.platform != "linux": + return + import ctypes + + # Define the prctl function with the appropriate arguments and return type + libc = ctypes.CDLL("libc.so.6") + prctl = libc.prctl + prctl.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_ulong, + ctypes.c_ulong, + ctypes.c_ulong, + ] + prctl.restype = ctypes.c_int + + # Set the process name to "my_process" + prctl(15, name.encode(), 0, 0, 0) + + +def _init_proc( + func: Callable, + out_file: Path, + proc_name: str, + on_except: Callable[[Exception, mp.process.BaseProcess], None] | None, + **kwargs: Any, +) -> None: + # Create a new process group + os.setsid() + + # Open stdout and stderr + with open(out_file, "w") as out_fd: + os.dup2(out_fd.fileno(), sys.stdout.fileno()) + os.dup2(out_fd.fileno(), sys.stderr.fileno()) + + # Print some information + pid = os.getpid() + gpid = os.getpgid(pid=pid) + + # Set the process name + _set_proc_name(proc_name) + + # Close stdin + sys.stdin.close() + + linebreak = "=" * 5 + # Execute the main function + print(linebreak + f" {func.__name__}:{pid} " + linebreak, file=sys.stderr) + try: + func(**kwargs) + except Exception as ex: + traceback.print_exc() + if on_except is not None: + on_except(ex, mp.current_process()) + + # Kill the new process and all its children by sending a SIGTERM signal to the process group + pid = os.getpid() + gpid = os.getpgid(pid=pid) + print(f"Killing process group pid={pid} gpid={gpid}", file=sys.stderr) + os.killpg(gpid, signal.SIGTERM) + sys.exit(1) + # Don't use a finally block here, because we want the exitcode to be set to + # 0 if the function returns normally + + +def spawn( + *, + out_file: Path, + on_except: Callable[[Exception, mp.process.BaseProcess], None] | None, + func: Callable, + **kwargs: Any, +) -> MPProcess: + # Decouple the process from the parent + if mp.get_start_method(allow_none=True) is None: + mp.set_start_method(method="forkserver") + + # Set names + proc_name = f"MPExec:{func.__name__}" + + # Start the process + proc = mp.Process( + target=_init_proc, + args=(func, out_file, proc_name, on_except), + name=proc_name, + kwargs=kwargs, + ) + proc.start() + + # Return the process + mp_proc = MPProcess(name=proc_name, proc=proc, out_file=out_file) + + return mp_proc diff --git a/pkgs/clan-app/pyproject.toml b/pkgs/clan-app/pyproject.toml index 1502af61f..96602631d 100644 --- a/pkgs/clan-app/pyproject.toml +++ b/pkgs/clan-app/pyproject.toml @@ -36,10 +36,6 @@ disallow_untyped_calls = true disallow_untyped_defs = true no_implicit_optional = true -[[tool.mypy.overrides]] -module = "argcomplete.*" -ignore_missing_imports = true - [[tool.mypy.overrides]] module = "clan_cli.*" ignore_missing_imports = true From 83c6945e39fb79636ed1ceac899ddc483ec0afb7 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Wed, 7 Aug 2024 15:50:30 +0200 Subject: [PATCH 2/4] clanModule: Init wifi iwd clan module, with which you can predefine wifi network credentials --- clanModules/flake-module.nix | 1 + clanModules/iwd/README.md | 10 +++++ clanModules/iwd/default.nix | 77 +++++++++++++++++++++++++++++++++ pkgs/installer/flake-module.nix | 21 +-------- 4 files changed, 90 insertions(+), 19 deletions(-) create mode 100644 clanModules/iwd/README.md create mode 100644 clanModules/iwd/default.nix diff --git a/clanModules/flake-module.nix b/clanModules/flake-module.nix index f385b87f3..3f13fc971 100644 --- a/clanModules/flake-module.nix +++ b/clanModules/flake-module.nix @@ -1,6 +1,7 @@ { ... }: { flake.clanModules = { + iwd = ./iwd; borgbackup = ./borgbackup; borgbackup-static = ./borgbackup-static; deltachat = ./deltachat; diff --git a/clanModules/iwd/README.md b/clanModules/iwd/README.md new file mode 100644 index 000000000..1ac1001fe --- /dev/null +++ b/clanModules/iwd/README.md @@ -0,0 +1,10 @@ +--- +description = "Automatically provisions wifi credentials" +--- + + +!!! Warning + This module is for demo purposes only right now the password is not encrypted and world readable! + + + diff --git a/clanModules/iwd/default.nix b/clanModules/iwd/default.nix new file mode 100644 index 000000000..6875dcdb9 --- /dev/null +++ b/clanModules/iwd/default.nix @@ -0,0 +1,77 @@ +{ lib, config, ... }: + +let + cfg = config.clan.iwd; + secret_path = ssid: config.clan.core.facts.services."iwd.${ssid}".secret."wifi-password".path or ""; + secret_generator = name: value: { + name = "iwd.${value.ssid}"; + value = { + secret."iwd.${value.ssid}" = { }; + generator.prompt = "Wifi password for '${value.ssid}'"; + generator.script = '' + config=" + [Security] + Passphrase=$prompt_value + " + echo "$config" > $secrets/wifi-password + ''; + }; + }; +in +{ + options.clan.iwd = { + networks = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, ... }: + { + options = { + ssid = lib.mkOption { + type = lib.types.strMatching "^[a-zA-Z0-9._-]+$"; + default = name; + description = "The name of the wifi network"; + }; + }; + } + ) + ); + default = { }; + description = "Wifi networks to predefine"; + }; + }; + + imports = [ + (lib.mkRemovedOptionModule [ + "clan" + "iwd" + "enable" + ] "Just define clan.iwd.networks to enable it") + ]; + + config = lib.mkMerge [ + (lib.mkIf (cfg.networks != { }) { + # Systemd tmpfiles rule to create /var/lib/iwd/example.psk file + systemd.tmpfiles.rules = lib.mapAttrsToList ( + _: value: "C /var/lib/iwd/${value.ssid}.psk 0600 root root - ${secret_path value.ssid}" + ) cfg.networks; + + clan.core.facts.services = lib.mapAttrs' secret_generator cfg.networks; + }) + { + # disable wpa supplicant + networking.wireless.enable = false; + + # Use iwd instead of wpa_supplicant. It has a user friendly CLI + networking.wireless.iwd = { + enable = true; + settings = { + Network = { + EnableIPv6 = true; + RoutePriorityOffset = 300; + }; + Settings.AutoConnect = true; + }; + }; + } + ]; +} diff --git a/pkgs/installer/flake-module.nix b/pkgs/installer/flake-module.nix index b00c22a79..def3b2b98 100644 --- a/pkgs/installer/flake-module.nix +++ b/pkgs/installer/flake-module.nix @@ -1,32 +1,15 @@ { self, lib, ... }: let - wifiModule = - { ... }: - { - # use iwd instead of wpa_supplicant - networking.wireless.enable = false; - - # Use iwd instead of wpa_supplicant. It has a user friendly CLI - networking.wireless.iwd = { - enable = true; - settings = { - Network = { - EnableIPv6 = true; - RoutePriorityOffset = 300; - }; - Settings.AutoConnect = true; - }; - }; - }; flashInstallerModule = { config, ... }: { imports = [ - wifiModule + self.clanModules.iwd self.nixosModules.installer ]; + system.stateVersion = config.system.nixos.version; nixpkgs.pkgs = self.inputs.nixpkgs.legacyPackages.x86_64-linux; } From d2d5ab4a9eb795b78f441ad1c90a6d9de8f34843 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Wed, 7 Aug 2024 15:54:36 +0200 Subject: [PATCH 3/4] formatter: Add prettier to format js files --- formatter.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/formatter.nix b/formatter.nix index c594cc73b..f9555b43b 100644 --- a/formatter.nix +++ b/formatter.nix @@ -43,6 +43,9 @@ "*.yaml" "*.yml" ]; + # plugins = [ + # "${self'.packages.prettier-plugin-tailwindcss}/lib/node_modules/prettier-plugin-tailwindcss/dist/index.mjs" + # ]; }; treefmt.programs.mypy.directories = { From 2dacbd5f3a7f7c0eb02060252c92eb2001830264 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Wed, 7 Aug 2024 16:04:46 +0200 Subject: [PATCH 4/4] docs: Fix missing nav link to iwd clan module --- clanModules/iwd/README.md | 4 ---- docs/mkdocs.yml | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/clanModules/iwd/README.md b/clanModules/iwd/README.md index 1ac1001fe..c33051abc 100644 --- a/clanModules/iwd/README.md +++ b/clanModules/iwd/README.md @@ -3,8 +3,4 @@ description = "Automatically provisions wifi credentials" --- -!!! Warning - This module is for demo purposes only right now the password is not encrypted and world readable! - - diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 443879c38..fa72fc6b0 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -58,6 +58,7 @@ nav: - reference/clanModules/borgbackup-static.md - reference/clanModules/borgbackup.md - reference/clanModules/deltachat.md + - reference/clanModules/iwd.md - reference/clanModules/ergochat.md - reference/clanModules/localbackup.md - reference/clanModules/localsend.md