From 10e8ccc890109b86a39bf355fe41702379971d22 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 19:51:13 +0000 Subject: [PATCH 01/47] docs(cookbook): config.toml [tools]/[jails] + cookbook + doc tooling Stage the documentation and configuration work split out of the larger get-a-vm-running branch into its own PR off main: - docs/cookbook: rewritten intro/install + new config.md, documenting the /etc/hyper/config.toml [tools] and [jails] tables, the [tools] node-tool paths, and the User Configuration / cgroup setup. - mix.exs/mix.lock: makeup_syntect + a docs alias step that aliases bash/sh fences to the shell grammar, so cookbook code blocks highlight. - config/runtime.exs: optional /etc/hyper/config.exs operator override. - lib/hyper/config.ex + umoci.ex: the [tools]/[jails] config reading these docs describe. Note: this branch is a content snapshot off main; it intentionally diverges from main's behavior (the supporting Rust/test changes live on the other branch), so it is not expected to build/test green on its own. --- config/runtime.exs | 47 +++-- docs/cookbook/config.md | 278 ++++++++++++++++++++++++++++++ docs/cookbook/install.md | 248 ++++++++++++++++++++++++++ docs/cookbook/intro.md | 52 +++--- lib/hyper/config.ex | 236 ++++++++++++++++++------- lib/hyper/img/oci_loader/umoci.ex | 18 +- mix.exs | 38 ++++ mix.lock | 3 + 8 files changed, 803 insertions(+), 117 deletions(-) create mode 100644 docs/cookbook/config.md create mode 100644 docs/cookbook/install.md diff --git a/config/runtime.exs b/config/runtime.exs index a3b4fd0f..45dbba68 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -14,17 +14,42 @@ config :hyper, Hyper.Node.Config.Budget, # Where to send traces. Defaults to Honeycomb; override OTEL_EXPORTER_OTLP_* # to point at any OTLP/HTTP backend (Collector, Grafana, etc). if config_env() != :test do - endpoint = System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT", "https://api.honeycomb.io") + custom_endpoint = System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT") + api_key = System.get_env("HONEYCOMB_API_KEY") - headers = - case System.get_env("HONEYCOMB_API_KEY") do - nil -> [] - "" -> [] - key -> [{"x-honeycomb-team", key}] - end + cond do + api_key not in [nil, ""] -> + config :opentelemetry_exporter, + otlp_protocol: :http_protobuf, + otlp_endpoint: custom_endpoint || "https://api.honeycomb.io", + otlp_headers: [{"x-honeycomb-team", api_key}] + + custom_endpoint not in [nil, ""] -> + # A custom OTLP backend (e.g. a local Collector) needs no Honeycomb key. + config :opentelemetry_exporter, + otlp_protocol: :http_protobuf, + otlp_endpoint: custom_endpoint, + otlp_headers: [] + + true -> + # No backend configured: exporting to the Honeycomb default with no key + # 401s on every batch. Stay silent instead (typical for local dev). Set + # HONEYCOMB_API_KEY or OTEL_EXPORTER_OTLP_ENDPOINT to enable tracing. + config :opentelemetry, traces_exporter: :none + end +end - config :opentelemetry_exporter, - otlp_protocol: :http_protobuf, - otlp_endpoint: endpoint, - otlp_headers: headers +# Operator overrides from a well-known location. An optional Elixir config file +# at /etc/hyper/config.exs (override the path with HYPER_CONFIG) is merged in +# last, so its values win over every default set above. An absent file is a +# no-op -- the normal case in dev and CI. Skipped under :test so the suite never +# reads host state. +if config_env() != :test do + hyper_config = System.get_env("HYPER_CONFIG") || "/etc/hyper/config.exs" + + if File.exists?(hyper_config) do + for {app, kw} <- Config.Reader.read!(hyper_config, env: config_env()) do + config app, kw + end + end end diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md new file mode 100644 index 00000000..cf59b7cf --- /dev/null +++ b/docs/cookbook/config.md @@ -0,0 +1,278 @@ +# Configuration + +Configuring `Hyper` is done through four layers, in priority: + + 1. Runtime `/etc/hyper/config.exs` is the canonical Elixir way to configure + the system. This allows you to inject arbitrary code to configure `Hyper`. + 2. `Hyper` will fall back to reading `/etc/hyper/config.toml` at runtime, on + bootup, on each node. + 3. `Hyper` will use its compile-time configuration through `config.ex`. + 4. `Hyper` will use defaults. + +**Note that not all layers allow all configuration fields to be tweaked.** This +is usually done for security. + +## Configuration Files + +### `/etc/hyper/config.exs` + +The `config.exs` file is exlusively used by the unprivileged `hyper` +application. The purpose of this file is to allow you to load configuration +values at runtime. If you are using a secrets manager, this is the right place +to load the secrets. + +### `/etc/hyper/config.toml` + +The `/etc/hyper/config.toml` file is used for static configuration. Unlike +`config.exs`, it is used by both `Hyper` and `hyper-suidhelper` which means +that it can impact the behavior of a process running under `root`. + +### Compile-Time Config + +The compile-time configuration is generally used to fine-tune the performance +of Hyper. You likely do not need to edit most of the configuration fields +exposed by this file for day-to-day usage, but they are available for you to +tweak. + +## Configuration Fields + +### Tool Configuration + +Hyper relies on a large number of external tools, all configured under the +`[tools]` table in `/etc/hyper/config.toml`. + +#### Privileged tools (run by the setuid helper) + +| Tool | Required | Default | `/etc/hyper/config.toml` | +|------|----------|---------|--------------------------| +| `firecracker` | Yes | - | `tools.firecracker` | +| `jailer` | Yes | - | `tools.jailer` | +| `dmsetup` | No | `/usr/sbin/dmsetup` | `tools.dmsetup` | +| `losetup` | No | `/usr/sbin/losetup` | `tools.losetup` | +| `blockdev` | No | `/usr/sbin/blockdev` | `tools.blockdev` | + +> #### Requirements {: .info} +> +> - These paths **can only** be configured through `/etc/hyper/config.toml`. +> Both `Hyper` and `hyper-setuidhelper` rely on these paths being identical. +> +> - The paths **must** be given as absolute paths. +> - The basename **must** match the configuration, eg. `firecracker` must have +> a path `/foo/bar/firecracker`. +> - The tools must be owned by the `root` user. +> - The tools must be exlusively writable by `root`. + +#### Node tools (run by the unprivileged node) + +| Tool | Required | Default | `/etc/hyper/config.toml` | +|------|----------|---------|--------------------------| +| `skopeo` | No | `skopeo` (on `PATH`) | `tools.skopeo` | +| `mke2fs` | No | `mke2fs` (on `PATH`) | `tools.mke2fs` | +| `umoci` | No | downloaded + cached under `/redist/umoci` | `tools.umoci` | +| `suidhelper` | No | `/usr/local/bin/hyper-suidhelper` | `tools.suidhelper` | + +> #### These are not privileged {: .info} +> +> The node runs these directly as the unprivileged hyper user, so — unlike the +> privileged tools above — they carry **no** root-ownership or basename +> requirement. `skopeo`/`mke2fs` default to the bare name resolved on `PATH`; +> leave `umoci` unset to let Hyper download and cache a pinned release. They +> share the one `[tools]` table with the privileged binaries — the helper simply +> ignores the keys it does not own. + +## The shared file: `/etc/hyper/config.toml` + +> #### Security {: .error} +> +> This file **must** be owned by `root` and be neither group- nor +> world-writable (e.g. mode `0644`). The setuid helper refuses to start +> otherwise — a present-but-untrusted file is treated as operator +> misconfiguration and is fatal (exit `2`), never silently ignored. + +### Root keys + +| Key | Type | Default | Meaning | +|-----|------|---------|---------| +| `work_dir` | string (absolute path) | `/srv/hyper` | Root of all node-local runtime state. Every other directory is derived from it. Must be an absolute path. Strongly recommended to sit on an NVMe drive. | + +The following directories are derived from `work_dir` and are **not** +independently configurable: + +| Path | Purpose | +|------|---------| +| `/jails` | Per-VM chroot directories | +| `/socks` | Per-VM control/gRPC sockets | +| `/scratch` | Per-VM copy-on-write writable layers | +| `/layers` | Read-only image layer store | +| `/redist` | Node-downloaded binaries (`vmlinux`, `umoci`) | + +### `[jails]` — confinement + +| Key | Type | Default | Meaning | +|-----|------|---------|---------| +| `cgroup` | string | `"hyper"` | Parent cgroup under which every VM cgroup is nested (passed to the jailer as `--parent-cgroup`). The operator must create `/sys/fs/cgroup/` and enable subtree control. | +| `uid_gid_range` | `[min, max]` | `[900000, 999999]` | UID/GID band each VM jail is allocated from. `min` must be `>= 1` and `<= max`; `min = 0` is rejected (uid 0 is root, and the jailer skips its privilege drop for uid 0). | + +> #### `uid_gid_range` is enforced on both sides {: .warning} +> +> The node only hands out UIDs in this range, and the helper only *accepts* +> UIDs in this range. Because both read the same file, narrowing the band is a +> single edit here — no second place to keep in sync. Nothing else on the host +> may use UIDs/GIDs in this range. + +### Complete example + +```toml +# Root of all node-local state. Strongly prefer an NVMe-backed mount. +work_dir = "/srv/hyper" + +# External binaries. The privileged ones (firecracker..blockdev) must be +# root-owned, not group/world-writable, absolute, and named exactly as their +# key; the node tools (skopeo/mke2fs/umoci/suidhelper) have no such requirement. +[tools] +firecracker = "/opt/firecracker/firecracker" # required; basename must be 'firecracker' +jailer = "/opt/firecracker/jailer" # required; basename must be 'jailer' +# dmsetup = "/usr/sbin/dmsetup" # optional (default shown) +# losetup = "/usr/sbin/losetup" # optional (default shown) +# blockdev = "/usr/sbin/blockdev" # optional (default shown) +# skopeo = "skopeo" # optional node tool (default shown) +# mke2fs = "mke2fs" # optional node tool (default shown) +# umoci = "/usr/bin/umoci" # optional; omit to auto-download +# suidhelper = "/usr/local/bin/hyper-suidhelper" # optional (default shown) + +[jails] +cgroup = "hyper" # default +uid_gid_range = [900000, 999999] # default +``` + +The minimal file is just `work_dir` plus the two required tools — everything +else defaults. + +## Node-only configuration (`config :hyper`) + +These have no helper counterpart and stay in `config :hyper`. The node's tool +paths (`skopeo`, `mke2fs`, `umoci`, `suidhelper`) used to live here but now read +from the `[tools]` table above — see [Tool Configuration](#tool-configuration). + +### Guest kernels + +| Key | Where read | Type | Default | Meaning | +|-----|-----------|------|---------|---------| +| `vmlinux` | runtime | `%{arch => path}` | `%{}` | Per-architecture guest kernel images, keyed by `Sys.Arch.t()`. The operator places kernels on the host and points these at them. | + +```elixir +config :hyper, + vmlinux: %{x86_64: "/srv/hyper/redist/vmlinux/vmlinux-x86_64"} +``` + +### Resource budget — `Hyper.Node.Config.Budget` + +The per-node resource budget. **Required**: the node refuses to boot if it is +absent. Set it in `config/runtime.exs`. Use the `Unit.*` quantities, never bare +numbers. + +| Key | Type | Meaning | +|-----|------|---------| +| `mem_max` | `Unit.Information.t()` | Hard memory cap for this node. | +| `disk_max` | `Unit.Information.t()` | Hard disk cap for this node. | +| `cpu_max_load` | float `0.0..1.0` | CPU-utilization fraction above which the node is considered full. | +| `disk_bw_cap` | `Unit.Bandwidth.t()` | Absolute disk throughput capacity. | +| `disk_bw_max_load` | float `0.0..1.0` | Fraction of `disk_bw_cap` past which disk is saturated. | +| `net_bw_cap` | `Unit.Bandwidth.t()` | Absolute network throughput capacity. | +| `net_bw_max_load` | float `0.0..1.0` | Fraction of `net_bw_cap` past which network is saturated. | + +```elixir +config :hyper, Hyper.Node.Config.Budget, + mem_max: Unit.Information.gib(4), + disk_max: Unit.Information.gib(4), + cpu_max_load: 0.8, + disk_bw_cap: Unit.Bandwidth.gibps(1), + disk_bw_max_load: 0.8, + net_bw_cap: Unit.Bandwidth.gibps(1), + net_bw_max_load: 0.8 +``` + +### gRPC server — `Hyper.Grpc.Config` + +The public gRPC interface. **Disabled by default.** + +| Key | Type | Default | Meaning | +|-----|------|---------|---------| +| `enabled` | boolean | `false` | Whether the server starts. | +| `port` | port number | `50051` | Listen port. | +| `cred` | `GRPC.Credential.t()` \| `nil` | `nil` | TLS credential, or `nil` for plaintext. | +| `adapter_opts` | keyword | `[]` | Forwarded to the server adapter, e.g. `[ip: {0, 0, 0, 0}]`. | + +```elixir +config :hyper, Hyper.Grpc.Config, + enabled: true, + port: 50_051, + cred: GRPC.Credential.new(ssl: [certfile: "/path/cert.pem", keyfile: "/path/key.pem"]) +``` + +> #### Co-located nodes {: .info} +> +> Every node binds `:port`. Running multiple nodes on one host requires giving +> each a distinct port. Build the TLS credential where you load your keys +> (e.g. `config/runtime.exs`); Hyper never reads the filesystem on your behalf. + +### Layer garbage collector — `Hyper.Img.Db.Gc.Config` + +A cluster-wide singleton that prunes unreferenced image layers. Every field has +a default; set only what you change. Durations are `Unit.Time` values, so +overrides belong in `config/runtime.exs`. Set `enabled: false` to never start it. + +| Key | Type | Default | Meaning | +|-----|------|---------|---------| +| `enabled` | boolean | `true` | Run the collector at all. | +| `batch_size` | `pos_integer` | `200` | Rows per keyset page (smaller = finer pause granularity). | +| `batch_pause` | `Unit.Time.t()` | `100ms` | Pause between pages within a sweep. | +| `sweep_interval` | `Unit.Time.t()` | `60s` | Rest between completed sweeps. | +| `acquire_interval` | `Unit.Time.t()` | `5s` | How often a standby retries to become the active singleton. | +| `retry` | `Unit.Time.t()` | `60s` | Backoff when the medium or DB is unavailable. | +| `statement_timeout` | `Unit.Time.t()` | `5s` | Cap on each GC DB statement so it can't pin a backend. | +| `grace_period` | `Unit.Time.t()` | `1h` | Never prune a blob younger than this (protects a row whose file is still being published). | + +```elixir +config :hyper, Hyper.Img.Db.Gc.Config, + enabled: true, + sweep_interval: Unit.Time.s(30), + grace_period: Unit.Time.s(60 * 60) +``` + +### Orphaned-resource reaper — `Hyper.Node.Reaper.Config` + +A per-node sweeper that reclaims orphaned firecracker cgroups and `hyper-rw-*` +device-mapper volumes left behind by unclean BEAM deaths. Uses a two-strike +grace period (an orphan must be seen on two consecutive ticks before it is +reaped). Set `enabled: false` to never start it. + +| Key | Type | Default | Meaning | +|-----|------|---------|---------| +| `enabled` | boolean | `true` | Run the reaper at all. | +| `interval` | `Unit.Time.t()` | `60s` | Rest between reap ticks. | + +```elixir +config :hyper, Hyper.Node.Reaper.Config, + enabled: true, + interval: Unit.Time.s(30) +``` + +### Telemetry (OpenTelemetry) + +Tracing is configured in `config/runtime.exs` from environment variables: + +| Variable | Effect | +|----------|--------| +| `HONEYCOMB_API_KEY` | Export to `https://api.honeycomb.io` with this key. | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | If `HONEYCOMB_API_KEY` is unset, export to this OTLP/HTTP endpoint (e.g. a local Collector), no auth header. | + +If neither is set, tracing is disabled. + +### Database and cluster topology + +The image-metadata database (`Hyper.Img.Db.Repo`, a standard Ecto/PostgreSQL +repo) and the cluster topology (`:libcluster`) are configured in +`config/config.exs` like any Elixir app. PostgreSQL is a required runtime +dependency — the node will not boot without a reachable instance. See +[Installation](install.md) for connection setup. diff --git a/docs/cookbook/install.md b/docs/cookbook/install.md new file mode 100644 index 00000000..8b22c877 --- /dev/null +++ b/docs/cookbook/install.md @@ -0,0 +1,248 @@ +# Quick Start + +This document provides the quickest start available to get Hyper running. + +## Configuration + +Before you can use `Hyper`, you must do a large amount of configuration. The +following guide must be applied on all nodes you run `Hyper` on. + +Before proceeding, ensure you meet all of these hard requirements: + +| Requirement | Test | +|-------------|------| +| [KVM](https://linux-kvm.org/page/Main_Page) available | `stat /dev/kvm` returns zero. | +| You have root access through `sudo`. | - | +| Your machine has cgroups V2 | `stat -fc %T /sys/fs/cgroup` returns zero. | + +### OS Packages + + + +### Ubuntu + +You can install the required packages by running: + +```sh +sudo apt update && sudo apt install -y \ + coreutils \ + e2fsprogs \ + libc-bin \ + linux-modules-extra-$(uname -r) \ + lvm2 \ + skopeo \ + util-linux +``` + +### Rocky + +You can install the required packages by running: + +```sh +sudo dnf install -y \ + coreutils \ + e2fsprogs \ + glibc-common \ + kernel-modules-extra-$(uname -r) \ + lvm2 \ + skopeo \ + util-linux +``` + +> #### Untested {: .warning} +> +> Rocky has not been tested, but should work. + + + +### Device Mapper Config + +Hyper relies on `dm-snapshot` and `dm-thin` to build COW filesystems. Load the +modules and confirm the targets are present: + +```sh +sudo modprobe dm_snapshot dm_thin_pool loop +sudo dmsetup targets # must list snapshot, thin, and thin-pool +``` + +> #### Persistent Config {: .warning} +> +> Loading modules via `modprobe` is ephemeral and will be reset on next boot. +> To make your config persistent: +> +> ```sh +> printf 'dm_snapshot\ndm_thin_pool\nloop\n' \ +> | sudo tee /etc/modules-load.d/hyper.conf +> ``` + +### PostgreSQL + +Hyper needs a **PostgreSQL** server reachable from every node - it is the image +database and the only stateful external dependency. + +For local development the quickest path is Docker. The connection details below +match the defaults in `config/config.exs` (`Hyper.Img.Db.Repo`): + +```sh +docker run -d --name hyper-pg \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=hyper_dev \ + -p 5432:5432 \ + postgres:16 +``` + +> #### Persistence {: .warning} +> +> Note that the example container should not be used in production -- it will +> be deleted on boot. +> +> We highly suggest you get a managed PostgresSQL instance. The following +> commonly used options are available: +> +> - [AWS RDS](https://aws.amazon.com/rds/postgresql/) if you're in the AWS +> ecosystem. +> - [GCP CloudSQL](https://cloud.google.com/sql) if you're in the GCP +> ecosystem. +> +> The author uses GCP. + +### Configuration + +It is mandatory that you create an `/etc/hyper/config.toml` file on every node. +A reasonable starting point is: + +```toml +# The working directory for hyper. Hyper will create a directory tree in this +# directory and running images, sockets and scratch space will be created in +# this directory. We **strongly** encourage this be mounted on an NVMe drive. +work_dir = "/srv/hyper" + +# Paths to every external binary hyper uses. All paths must be absolute. +# +# The privileged binaries the setuid helper runs (firecracker, jailer, dmsetup, +# losetup, blockdev) must be root-owned and not group/world writable -- the +# helper refuses them otherwise. The node-run tools (skopeo, umoci, mke2fs) have +# no such requirement. +[tools] +# **required**. basename **must** be 'firecracker'. +firecracker = "/opt/firecracker/firecracker" + +# **required**. basename **must** be 'jailer'. +jailer = "/opt/firecracker/jailer" + +# optional -- privileged device tools, default to /usr/sbin/. +# dmsetup = "/usr/sbin/dmsetup" +# losetup = "/usr/sbin/losetup" +# blockdev = "/usr/sbin/blockdev" + +# optional -- node-run tools. skopeo/mke2fs default to the name on PATH; omit +# umoci to let hyper download and cache a pinned release. +# skopeo = "skopeo" +# mke2fs = "mke2fs" +# umoci = "/usr/bin/umoci" +# suidhelper = "/usr/local/bin/hyper-suidhelper" + +[jails] +# The valid range of user/group IDs in which new VMs will be spawned. Hyper +# will create new VM jails for each VM within the given range. +uid_gid_range = [900000, 999999] +# optional +cgroup = "hyper" +``` + +> #### Security {: .error} +> +> This file **must** be owned by `root`, not group and not world writable. +> `Hyper` will refuse to boot otherwise. + +For more details on configuring and tuning Hyper, we suggest you see the +[configuration guide](config.md). + +### Cgroups + +Hyper uses cgroups to impose limits on each VM. Each VM has its own cgroup, +which is spawned ephemerally, for the lifetime of the VM. These cgroups are all +managed by a parent cgroup which you must create. You can name this cgroup +whatever you like, as long as it matches the `jails.cgroup` value in the +`/etc/hyper/config.toml`: + +```sh +sudo mkdir -p /sys/fs/cgroup/hyper +``` + +You must allow permissions on `cpu` and `memory` control on the subtree: + +```sh +echo '+cpu +memory' | sudo tee /sys/fs/cgroup/hyper/cgroup.subtree_control +``` + +> #### Security {: .error} +> +> Note that Hyper does not manage the `cgroup` with its user -- it rather +> delegates to `hyper-suidhelper`, which is why `/sys/fs/cgroup/hyper` should +> be `root:root` owned. + +> #### Persistence {: .warning} +> +> The configuration, as given, will not survive reboots. To persist it, you can +> use `systemd-tempfiles`: +> +> ```sh +> echo 'd /sys/fs/cgroup/hyper 0755 root root -' \ +> | sudo tee /etc/tmpfiles.d/hyper-cgroup.conf +> ``` + +### User Configuration + +Hyper must **not** run as `root`, and you should not run it as your login user +either. Instead, give it a dedicated, unprivileged system user. The BEAM runs +as this user; every operation that genuinely needs root is routed through the +setuid helper (see [SUID Helper](#suid-helper)), so the node itself never holds +privilege. + +Create the user — system account, no login shell: + +```sh +sudo useradd --system --shell /usr/sbin/nologin --home-dir /srv/hyper hyper +``` + +Start Hyper as this user (for example `sudo -u hyper ...`, or `User=hyper` in a +systemd unit). The rest of this section covers the few permissions it needs — +and the ones it deliberately does **not**. + +#### Working directory + +The node builds its entire on-disk tree (`jails`, `socks`, `scratch`, `layers`, +`redist`) under `work_dir` (from `/etc/hyper/config.toml`, default `/srv/hyper`) +**as this user**. It must therefore own that directory: + +```sh +sudo mkdir -p /srv/hyper +sudo chown hyper:hyper /srv/hyper +``` + +## Installation + +### SUID Helper + +Hyper does not run as `root`. Running Hyper as root is considered unsafe and an +anti-pattern. Unfortunately, Hyper needs root for certain classes of system +operations. This is achieved through a side-car binary called +`hyper-setuidhelper`, which you must install manually. + +> #### Versioning {: .warning} +> +> The `hyper-setuidhelper` binary is versioned together with the version of +> `Hyper`, meaning that mismatched versions between the `hyper-setuidhelper` +> and `Hyper` itself will not work and Hyper will fail to boot. + +Installing this binary can be done by downloading it from the [Github Releases +page](https://github.com/harmont-dev/hyper/releases) and executing: + +```sh +sudo install -o root -g root -m 4755 \ + path/to/downloaded/hyper-suidehelper \ + /usr/local/bin/hyper-suidhelper +``` + diff --git a/docs/cookbook/intro.md b/docs/cookbook/intro.md index a0030d35..04500feb 100644 --- a/docs/cookbook/intro.md +++ b/docs/cookbook/intro.md @@ -13,19 +13,16 @@ of the aforementioned systems. The absolute best way to understand `Hyper` and how it works is to play around with it. -## Getting Started +#### Auto-redistributed -The absolute best way to get started with `Hyper` is to play with it. +`umoci` and the guest `vmlinux` kernels are downloaded, checksum-verified, and +managed by Hyper itself; you do not install them. -### Requirements - -Hyper requires the following software be installed on each node running it: - - - [`skopeo`](https://github.com/containers/skopeo) - - [`e2fsprogs`](https://github.com/tytso/e2fsprogs) - -Hyper has more runtime dependencies, but they are automatically redistributed -by Hyper. +`firecracker` and `jailer` are not auto-downloaded. Install them with +`mix firecracker.install [--prefix ]` (default prefix `/opt/firecracker`), +which downloads the pinned v1.16.0 release, places the binaries at +`/firecracker` and `/jailer`, and prints the `/etc/hyper/config.toml` +snippet to paste in. ### Installation @@ -33,31 +30,24 @@ by Hyper. ### Configuration -Running `Hyper` is involved and requires a large number of pre-requisites. The -configuration of `:hyper` can be done by creating a `config :hyper` entry in -your `config.exs`. Refer to the given snippet for details on each -configuration. +Almost all host configuration — `work_dir`, the `[tools]` binary paths +(`firecracker`, `jailer`, `dmsetup`, ...), and the `[jails]` table (`cgroup`, +`uid_gid_range`) — lives in `/etc/hyper/config.toml` (the single source of +truth shared with the setuid helper, shown above), and every node-local path +(`jails`, `socks`, `scratch`, `layers`) is derived from `work_dir`. None of it is +repeated in `config :hyper`. + +The node's own tool paths (`skopeo`, `mke2fs`, `umoci`, `suidhelper`) now live in +the `[tools]` table of `/etc/hyper/config.toml` alongside the privileged binaries, +so `config :hyper` holds only the per-architecture guest kernels (each with a +default, so the block may be omitted): ```elixir config :hyper, - # TODO(markovejnovic): Remove this after it gets auto-downloaded. - jailer_bin: "/opt/firecracker/jailer-v1.16.0-x86_64", - # TODO(markovejnovic): Remove this after it gets auto-downloaded. - firecracker_bin: "/opt/firecracker/firecracker-v1.16.0-x86_64", - # You must create a parent cgroup on your system. Continue reading for - # further details. - cgroup_parent: "hyper", - # TODO(markovejnovic): Merge these directories into one. - jailer_chroot_base: "/srv/hyper/jails", - socket_dir: "/srv/hyper/socks", - scratch_dir: "/srv/hyper/scratch", - # Hyper requires that each VM you pass - uid_gid_range: {900_000, 999_999}, - layer_dir: "/srv/hyper/layers" + # Per-architecture guest kernel images placed on the host. + vmlinux: %{x86_64: "/srv/hyper/redist/vmlinux/vmlinux-x86_64"} ``` - - ### Usage diff --git a/lib/hyper/config.ex b/lib/hyper/config.ex index 1332d9ee..b39ae96b 100644 --- a/lib/hyper/config.ex +++ b/lib/hyper/config.ex @@ -1,64 +1,148 @@ defmodule Hyper.Config do @moduledoc """ - Host configuration, read from `config :hyper, ...` (see `config/config.exs`). + Host configuration. - `work_dir` is the one value shared with the setuid helper - (`native/suidhelper`); both sides read it from `/etc/hyper/config.toml` so the - data root has a single source of truth (see `work_dir/0`). Everything else is - compile-time. + Everything shared with the setuid helper (`native/suidhelper`) is read from the + single source of truth, `/etc/hyper/config.toml`, at runtime — never duplicated + in `config :hyper`. The node and the helper parse the same file, so they cannot + drift: `work_dir`, the `[tools]` binary paths (`firecracker`, `jailer`, ...), + and the `[jails]` table (`cgroup`, `uid_gid_range`). The file is read once on first access + and cached in `:persistent_term`; an absent file (local dev / CI) yields the + same built-in defaults the helper compiles in, so both sides still agree. + + The node's own tools (`skopeo`, `umoci`, `mke2fs`, `suidhelper`) share that same + `[tools]` table — the helper simply ignores the keys it does not recognise, so + one table serves both. Only `vmlinux` and the cluster topology remain in + `config :hyper`. """ - # The shared data-root config file, read by both this node and the setuid - # helper. Absent in local dev / CI, where `@dev_work_dir` is used instead. + # The shared config file, read by both this node and the setuid helper. Absent + # in local dev / CI, where the built-in defaults below are used instead. @config_path "/etc/hyper/config.toml" @dev_work_dir "/srv/hyper" - @parent_cgroup Application.compile_env(:hyper, :cgroup_parent, "hyper") - @uid_gid_range Application.compile_env!(:hyper, :uid_gid_range) - @layer_dir Application.compile_env!(:hyper, :layer_dir) - @losetup_path Application.compile_env(:hyper, :losetup_path, "losetup") - @dmsetup_path Application.compile_env(:hyper, :dmsetup_path, "dmsetup") - @blockdev_path Application.compile_env(:hyper, :blockdev_path, "blockdev") - @skopeo_path Application.compile_env(:hyper, :skopeo_path, "skopeo") - @umoci_path Application.compile_env(:hyper, :umoci_path, nil) - @mke2fs_path Application.compile_env(:hyper, :mke2fs_path, "mke2fs") + # Defaults for the helper-shared values, kept in lockstep with the helper's + # `Config::default` (native/suidhelper/src/config.rs) so an absent config.toml + # makes the node and the helper agree out of the box. + @default_parent_cgroup "hyper" + @default_uid_gid_range {900_000, 999_999} + + # Defaults for the node-only `[tools]` binaries (skopeo/umoci/mke2fs/suidhelper). + # These bare keys live alongside the helper's own tools in the `[tools]` table; + # the helper ignores the keys it does not recognise, so the two sides share one + # table without colliding. + @default_skopeo "skopeo" + @default_mke2fs "mke2fs" + @default_suid_helper "/usr/local/bin/hyper-suidhelper" @doc """ Root work directory for this node. All firecracker paths derive from it. Read from `#{@config_path}` (the single source of truth shared with the setuid - helper) the first time it is needed, then cached in `:persistent_term`. Falls - back to `#{@dev_work_dir}` when the file is absent (local dev / CI, where the - helper is not installed anyway). + helper) the first time it is needed, then cached via `config_toml/0`. Falls back + to `#{@dev_work_dir}` when the file is absent (local dev / CI, where the helper + is not installed anyway). """ @spec work_dir :: Path.t() - def work_dir do - case :persistent_term.get({__MODULE__, :work_dir}, nil) do - nil -> - work_dir = load_work_dir() - :persistent_term.put({__MODULE__, :work_dir}, work_dir) - work_dir + def work_dir, do: Map.get(config_toml(), "work_dir", @dev_work_dir) + + @doc "Directory holding redistributable binaries downloaded by the node." + @spec redist_dir :: Path.t() + def redist_dir, do: Path.join(work_dir(), "redist") - work_dir -> - work_dir + @doc """ + Absolute path to the firecracker binary, from the `[tools]` table in + `#{@config_path}`. Raises if absent — the operator must configure it; there is + no default. + + For the launch path only. Pre-launch checks should use `firecracker_bin_configured/0` + so a missing key returns a typed error rather than crashing. + """ + @spec firecracker_bin :: Path.t() + def firecracker_bin, do: fetch_tool!("firecracker") + + @doc """ + Non-raising form of `firecracker_bin/0`. Returns `{:ok, path}` when the + `[tools] firecracker` key is present in `#{@config_path}`, or `:error` otherwise. + """ + @spec firecracker_bin_configured :: {:ok, Path.t()} | :error + def firecracker_bin_configured, do: Map.fetch(tools(), "firecracker") + + @doc """ + Absolute path to the jailer binary, from the `[tools]` table in `#{@config_path}`. + Raises if absent — the operator must configure it; there is no default. + + For the launch path only. Pre-launch checks should use `jailer_bin_configured/0` + so a missing key returns a typed error rather than crashing. + """ + @spec jailer_bin :: Path.t() + def jailer_bin, do: fetch_tool!("jailer") + + @doc """ + Non-raising form of `jailer_bin/0`. Returns `{:ok, path}` when the + `[tools] jailer` key is present in `#{@config_path}`, or `:error` otherwise. + """ + @spec jailer_bin_configured :: {:ok, Path.t()} | :error + def jailer_bin_configured, do: Map.fetch(tools(), "jailer") + + # The `[tools]` table (binary paths shared with the helper), or `%{}` when the + # file or table is absent. + @spec tools :: map() + defp tools, do: Map.get(config_toml(), "tools", %{}) + + @spec fetch_tool!(String.t()) :: Path.t() + defp fetch_tool!(key) do + case Map.fetch(tools(), key) do + {:ok, path} -> + path + + :error -> + raise "#{@config_path}: `[tools] #{key}` is not set; " <> + "operator must configure it before starting the node" end end - @spec load_work_dir :: Path.t() - defp load_work_dir do - case File.read(@config_path) do - {:ok, body} -> body |> Toml.decode!() |> Map.fetch!("work_dir") - {:error, _} -> @dev_work_dir + # A `[tools]` path with a built-in default: the configured string, or `default` + # when the key is absent (or set to a non-string, treated as unset). The + # `is_binary/1` guard pins the success type to `String.t()` so the public + # accessors stay precisely typed for Dialyzer rather than widening to `any()`. + @spec tool_path(String.t(), String.t()) :: String.t() + defp tool_path(key, default) do + case Map.get(tools(), key) do + path when is_binary(path) -> path + _ -> default end end - @doc "Directory holding redistributable binaries downloaded by the node." - @spec redist_dir :: Path.t() - def redist_dir, do: Path.join(work_dir(), "redist") + # A `[tools]` path with no default: the configured string, or `nil` when unset. + @spec optional_tool_path(String.t()) :: String.t() | nil + defp optional_tool_path(key) do + case Map.get(tools(), key) do + path when is_binary(path) -> path + _ -> nil + end + end + + @spec config_toml :: map() + defp config_toml do + case :persistent_term.get({__MODULE__, :config_toml}, nil) do + nil -> + cfg = load_config_toml() + :persistent_term.put({__MODULE__, :config_toml}, cfg) + cfg - @doc "Directory where `Hyper.Node.FireVMM.Provider` installs the firecracker release." - @spec firecracker_install_dir :: Path.t() - def firecracker_install_dir, do: Path.join(redist_dir(), "firecracker") + cfg -> + cfg + end + end + + @spec load_config_toml :: map() + defp load_config_toml do + case File.read(@config_path) do + {:ok, body} -> Toml.decode!(body) + {:error, _} -> %{} + end + end @doc "Directory where `Hyper.Node.FireVMM.VmLinux.Provider` installs guest kernels." @spec vmlinux_install_dir :: Path.t() @@ -77,9 +161,16 @@ defmodule Hyper.Config do def chroot_base, do: Path.join(work_dir(), "jails") @doc """ - A name for the parent cgroup which is used as a supervision cgroup for all VMs. + Name of the parent cgroup used as a supervision cgroup for all VMs. Read from + `[jails] cgroup` in `#{@config_path}` (shared with the helper), default `"hyper"`. """ - def parent_cgroup, do: @parent_cgroup + @spec parent_cgroup :: String.t() + def parent_cgroup, do: Map.get(jails(), "cgroup", @default_parent_cgroup) + + # The `[jails]` table (VM placement/confinement, shared with the helper), or + # `%{}` when the file or table is absent. + @spec jails :: map() + defp jails, do: Map.get(config_toml(), "jails", %{}) @doc """ Path to the directory where all VM sockets are held. @@ -91,12 +182,20 @@ defmodule Hyper.Config do def socket_dir, do: Path.join(work_dir(), "socks") @doc """ - Range in which `Hyper` will attempt to allocate uid/gids. Whenever a VM is allocated, it will - get a fresh uid/gid pair in this range. It is absolutely critical that this range is not used - by any other process on the system, as that can risk security. + Range in which `Hyper` allocates uid/gids: each VM gets a fresh uid/gid pair in + this range. Critical that no other process on the system uses this range. + + Read from `[jails] uid_gid_range` (a `[min, max]` array) in `#{@config_path}` — + the same file the helper validates against, so the node only ever hands out uids + the helper will accept. Defaults to `#{inspect(@default_uid_gid_range)}` when absent. """ @spec uid_gid_range :: {integer(), integer()} - def uid_gid_range, do: @uid_gid_range + def uid_gid_range do + case Map.get(jails(), "uid_gid_range") do + [min, max] -> {min, max} + _ -> @default_uid_gid_range + end + end @doc """ Location of all image layers on all nodes. @@ -107,40 +206,45 @@ defmodule Hyper.Config do Must be stable across all nodes, and must be a directory. If it does not exist, `Hyper.Node` will attempt to create one. + + Derived as `/layers`, so it follows `work_dir` from `#{@config_path}`. """ @spec layer_dir :: Path.t() - def layer_dir, do: @layer_dir - - @doc "Path to the losetup binary." - def losetup_path, do: @losetup_path + def layer_dir, do: Path.join(work_dir(), "layers") - @doc "Path to the dmsetup binary." - def dmsetup_path, do: @dmsetup_path - - @doc "Path to the blockdev binary." - def blockdev_path, do: @blockdev_path - - @doc "Path to the skopeo binary (used by `Hyper.Img.OciLoader` to pull OCI images)." - def skopeo_path, do: @skopeo_path + @doc """ + Path to the skopeo binary (used by `Hyper.Img.OciLoader` to pull OCI images). + Read from `[tools] skopeo` in `#{@config_path}`, default `#{@default_skopeo}` (on `PATH`). + """ + @spec skopeo_path :: String.t() + def skopeo_path, do: tool_path("skopeo", @default_skopeo) @doc """ Operator-configured path to the umoci binary, or `nil` (the default) to let - `Hyper.Img.OciLoader.Umoci` download and manage a pinned default. + `Hyper.Img.OciLoader.Umoci` download and manage a pinned default. Read from + `[tools] umoci` in `#{@config_path}`. """ - def umoci_path, do: @umoci_path + @spec umoci_path :: String.t() | nil + def umoci_path, do: optional_tool_path("umoci") - @doc "Path to the mke2fs binary (used by `Hyper.Img.OciLoader` to build the ext4 rootfs)." - def mke2fs_path, do: @mke2fs_path + @doc """ + Path to the mke2fs binary (used by `Hyper.Img.OciLoader` to build the ext4 rootfs). + Read from `[tools] mke2fs` in `#{@config_path}`, default `#{@default_mke2fs}` (on `PATH`). + """ + @spec mke2fs_path :: String.t() + def mke2fs_path, do: tool_path("mke2fs", @default_mke2fs) @doc """ - Path to the setuid-root device helper (`hyper-suidhelper`). Required: the node - runs unprivileged and routes every `losetup`/`dmsetup`/`blockdev` operation - through it. + Path to the setuid-root device helper (`hyper-suidhelper`). The node runs + unprivileged and routes every `losetup`/`dmsetup`/`blockdev` operation through + it. - Runtime config (host-specific), so it can be set per node without recompiling. + Read from `[tools] suidhelper` in `#{@config_path}`, default `#{@default_suid_helper}` + (the install path used by `mix suidhelper.install`), so an operator who installs + it elsewhere can override it per node without recompiling. """ - @spec suid_helper :: Path.t() - def suid_helper, do: Application.fetch_env!(:hyper, :suid_helper) + @spec suid_helper :: String.t() + def suid_helper, do: tool_path("suidhelper", @default_suid_helper) @doc """ Directory for per-VM scratch (writable-layer COW) files. Must be node-local and diff --git a/lib/hyper/img/oci_loader/umoci.ex b/lib/hyper/img/oci_loader/umoci.ex index 06929b58..dae13585 100644 --- a/lib/hyper/img/oci_loader/umoci.ex +++ b/lib/hyper/img/oci_loader/umoci.ex @@ -5,9 +5,9 @@ defmodule Hyper.Img.OciLoader.Umoci do Two sources, in priority order (mirrors `Hyper.Node.Vmlinux`): - 1. An operator-configured path via `config :hyper, umoci_path: - "/path/to/umoci"` (`Hyper.Config.umoci_path/0`). If set, it wins and is - never downloaded. + 1. An operator-configured path via `[tools] umoci` in + `/etc/hyper/config.toml` (`Hyper.Config.umoci_path/0`). If set, it wins + and is never downloaded. 2. Otherwise the pinned static binary downloaded by `ensure_installed/0` into `Hyper.Config.umoci_install_dir/0` (`/redist/umoci`). """ @@ -57,13 +57,13 @@ defmodule Hyper.Img.OciLoader.Umoci do """ @spec bin() :: Path.t() def bin do - configured = Config.umoci_path() + case Config.umoci_path() do + nil -> + {:ok, arch} = Sys.Arch.current() + default_path(arch) - if configured != nil do - configured - else - {:ok, arch} = Sys.Arch.current() - default_path(arch) + configured -> + configured end end diff --git a/mix.exs b/mix.exs index 7906c6a1..7ccae729 100644 --- a/mix.exs +++ b/mix.exs @@ -67,6 +67,10 @@ defmodule Hyper.MixProject do {:junit_formatter, "~> 3.4", only: :test, runtime: false}, {:dialyxir, "~> 1.4", only: [:dev], runtime: false}, {:ex_doc, "~> 0.34", only: :dev, runtime: false}, + # Syntect-backed Makeup lexer: covers the doc languages that have no + # dedicated Makeup lexer (markdown, toml, bash, sh, python). Elixir/erlang + # still use their native lexers; this fills the rest in one dep. + {:makeup_syntect, "~> 0.1", only: :dev, runtime: false}, {:ecto_sql, "~> 3.13"}, {:grpc, "~> 1.0"}, {:grpc_server, "~> 1.0"}, @@ -105,6 +109,8 @@ defmodule Hyper.MixProject do extras: [ "README.md", "docs/cookbook/intro.md", + "docs/cookbook/install.md", + "docs/cookbook/config.md", "docs/cookbook/architecture.md", "docs/grpc.md" ], @@ -183,6 +189,32 @@ defmodule Hyper.MixProject do end # `mix check` - the strict gate. Runs fast checks first, slow ones (dialyzer) last. + # Give ```bash / ```sh real syntax highlighting in the docs. + # + # Two obstacles, both about *who registers the lexer last*: + # 1. makeup_syntect registers the shell grammar only under its raw syntect + # name "Shell-Unix-Generic" (ExDoc resolves fences by lexer name, not + # file extension), so ```bash / ```sh never reach it. + # 2. ExDoc itself registers a minimal `ExDoc.ShellLexer` for sh/bash/shell/ + # zsh (it only de-selects the `$ ` prompt; everything else is plain text) + # from `ExDoc.Application.start`, which runs during the `docs` task. + # + # So we start :ex_doc and :makeup_syntect here first (idempotent — the later + # `docs` task won't re-run their `start/2`), then register our shell aliases + # LAST so they win. Dev-only; runs as the `docs` alias's first step. + defp register_doc_lexers(_args) do + {:ok, _} = Application.ensure_all_started(:makeup_syntect) + {:ok, _} = Application.ensure_all_started(:ex_doc) + + Makeup.Registry.register_lexer(MakeupSyntect.Lexer, + options: [language: "Shell-Unix-Generic"], + names: ["bash", "sh", "shell", "zsh"], + extensions: [] + ) + + :ok + end + defp aliases do [ check: [ @@ -192,6 +224,12 @@ defmodule Hyper.MixProject do "test --warnings-as-errors", "dialyzer" ], + # makeup_syntect registers the shell grammar only under its raw syntect + # name "Shell-Unix-Generic", and ExDoc resolves fences by lexer *name* + # (not file extension), so ```bash / ```sh would fall back to plain text. + # Alias them to the shell grammar before ExDoc runs (same VM, so the + # registration is visible to the highlighter). + docs: ["loadpaths", ®ister_doc_lexers/1, "docs"], # Force a regeneration of the Firecracker bindings (ignores staleness). "firecracker.gen": ["compile.firecracker_gen --force"], # Force a regeneration of the gRPC bindings (ignores staleness). diff --git a/mix.lock b/mix.lock index 06780d8a..8012573e 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,7 @@ %{ "acceptor_pool": {:hex, :acceptor_pool, "1.0.1", "d88c2e8a0be9216cf513fbcd3e5a4beb36bee3ff4168e85d6152c6f899359cdb", [:rebar3], [], "hexpm", "f172f3d74513e8edd445c257d596fc84dbdd56d2c6fa287434269648ae5a421e"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, + "castore": {:hex, :castore, "1.0.19", "6903cabdfd9d1af46454126e7c8385186659dd33ecfb74a885cae52221ad6109", [:mix], [], "hexpm", "3669e6cab13f54c2df26b3e6833745d647f35b6e30d8ddd5975df0d5c842ca98"}, "chatterbox": {:hex, :ts_chatterbox, "0.15.1", "5cac4d15dd7ad61fc3c4415ce4826fc563d4643dee897a558ec4ea0b1c835c9c", [:rebar3], [{:hpack, "~> 0.3.0", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "4f75b91451338bc0da5f52f3480fa6ef6e3a2aeecfc33686d6b3d0a0948f31aa"}, "cowboy": {:hex, :cowboy, "2.16.1", "fa04080b602ff25c40a7700f2dc0152dbc1ba26b42093ae0fa9bb7a337d5a242", [:make, :rebar3], [{:cowlib, ">= 2.16.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "b8ea4dd317a043e3177ec840cfa3bcb47cfb41035d3abb24d954dc7d51def399"}, "cowlib": {:hex, :cowlib, "2.17.1", "3e6053016d1ab245730f0af688755476dcedb1c25ed8fb5751f59a2bfdc0c9af", [:make, :rebar3], [], "hexpm", "ff08bd17e6dd931445b18af77315b9b5fe052407110964ad2588c686b57b5e3f"}, @@ -38,6 +39,7 @@ "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, + "makeup_syntect": {:hex, :makeup_syntect, "0.1.4", "e1230c9e0513c667b226b21c83eb182e1ab581f65af9441edab1f9ac626acba6", [:mix], [{:makeup, "~> 1.2", [hex: :makeup, repo: "hexpm", optional: false]}, {:rustler, "~> 0.37.1", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.8.2", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "5b624a434d9665786b9a352a5f3502b6c98e1996ede9936b20035ec140daef70"}, "merkle_map": {:hex, :merkle_map, "0.2.2", "f36ff730cca1f2658e317a3c73406f50bbf5ac8aff54cf837d7ca2069a6e251c", [:mix], [], "hexpm", "383107f0503f230ac9175e0631647c424efd027e89ea65ab5ea12eeb54257aaf"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mint": {:hex, :mint, "1.9.0", "d6f534c2a3e98b2a8cc749b4796eb77e9e3af79a76f96e4c74035a827de0d318", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "007154c7d8c43916aed3c93afd1f11aebbaa9c5ff4b7ba55ebe0d17ee0296042"}, @@ -57,6 +59,7 @@ "protobuf": {:hex, :protobuf, "0.17.0", "39e24e43c9648e148feba16ed51100b5b2028ea900b55460377b0476f6e10613", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "ca6c91f6f63e2c147b47f03eefd10b80538aa6fc55ff4b12b795efb786b0152f"}, "ranch": {:hex, :ranch, "2.2.0", "25528f82bc8d7c6152c57666ca99ec716510fe0925cb188172f41ce93117b1b0", [:make, :rebar3], [], "hexpm", "fa0b99a1780c80218a4197a59ea8d3bdae32fbff7e88527d7d8a4787eff4f8e7"}, "req": {:hex, :req, "0.6.2", "b9b2024f35bcf60a92cc8cad2eaaf9d4e7aace463ff74be1afe5986830184413", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cc9cd30a2ddd04989929b887178e1610c940456d962c6c3a52df6146d2eef9bf"}, + "rustler_precompiled": {:hex, :rustler_precompiled, "0.8.4", "700a878312acfac79fb6c572bb8b57f5aae05fe1cf70d34b5974850bbf2c05bf", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "3b33d99b540b15f142ba47944f7a163a25069f6d608783c321029bc1ffb09514"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, From b676e886aa93beb08d0a64f6610313c6e7ec720f Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 20:21:21 +0000 Subject: [PATCH 02/47] feat(cfg): internal TOML reader with dotted-path fetch --- lib/hyper/cfg/toml.ex | 61 ++++++++++++++++++++++++++++++++++++ test/hyper/cfg/toml_test.exs | 28 +++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 lib/hyper/cfg/toml.ex create mode 100644 test/hyper/cfg/toml_test.exs diff --git a/lib/hyper/cfg/toml.ex b/lib/hyper/cfg/toml.ex new file mode 100644 index 00000000..9ca878c0 --- /dev/null +++ b/lib/hyper/cfg/toml.ex @@ -0,0 +1,61 @@ +defmodule Hyper.Cfg.Toml do + @moduledoc false + # Internal: the raw read+cache of /etc/hyper/config.toml, the single source of + # truth shared with the setuid helper (native/suidhelper). Read once on first + # access and cached in :persistent_term; an absent file (local dev / CI) yields + # an empty map so the built-in defaults in Hyper.Cfg.* take over and the node + # still agrees with the helper. Only Hyper.Cfg.* may call this module. + + @config_path "/etc/hyper/config.toml" + + @doc "Fetch a dotted key path (e.g. `\"tools.firecracker\"`) from the cached config." + @spec fetch(String.t()) :: {:ok, term()} | :error + def fetch(path), do: fetch_in(config(), path) + + @doc "Pure dotted-path lookup into an already-decoded map (exposed for tests)." + @spec fetch_in(map(), String.t()) :: {:ok, term()} | :error + def fetch_in(map, path) do + Enum.reduce_while(String.split(path, "."), {:ok, map}, fn seg, {:ok, acc} -> + case acc do + %{^seg => v} -> {:cont, {:ok, v}} + _ -> {:halt, :error} + end + end) + end + + @doc "Path to the shared config file." + @spec path :: Path.t() + def path, do: @config_path + + @doc "Drop the cache so the next read re-parses the file (test hook)." + @spec reload :: map() + def reload do + :persistent_term.erase({__MODULE__, :config}) + config() + end + + @doc "Seed the cache with a decoded map, bypassing the file (test hook)." + @spec put_cache(map()) :: :ok + def put_cache(cfg), do: :persistent_term.put({__MODULE__, :config}, cfg) + + @spec config :: map() + defp config do + case :persistent_term.get({__MODULE__, :config}, nil) do + nil -> + cfg = load() + :persistent_term.put({__MODULE__, :config}, cfg) + cfg + + cfg -> + cfg + end + end + + @spec load :: map() + defp load do + case File.read(@config_path) do + {:ok, body} -> Toml.decode!(body) + {:error, _} -> %{} + end + end +end diff --git a/test/hyper/cfg/toml_test.exs b/test/hyper/cfg/toml_test.exs new file mode 100644 index 00000000..046e85e1 --- /dev/null +++ b/test/hyper/cfg/toml_test.exs @@ -0,0 +1,28 @@ +defmodule Hyper.Cfg.TomlTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.Toml + + setup do + on_exit(fn -> Toml.reload() end) + :ok + end + + test "fetch_in/2 traverses nested tables and stops at a missing segment" do + cfg = %{"tools" => %{"firecracker" => "/opt/fc"}} + assert Toml.fetch_in(cfg, "tools.firecracker") == {:ok, "/opt/fc"} + assert Toml.fetch_in(cfg, "tools.jailer") == :error + assert Toml.fetch_in(cfg, "tools.firecracker.extra") == :error + assert Toml.fetch_in(cfg, "missing") == :error + end + + test "fetch/1 reads the seeded cache; absent keys return :error" do + Toml.put_cache(%{"work_dir" => "/data", "tools" => %{"firecracker" => "/opt/fc"}}) + assert Toml.fetch("work_dir") == {:ok, "/data"} + assert Toml.fetch("tools.firecracker") == {:ok, "/opt/fc"} + assert Toml.fetch("tools.jailer") == :error + + Toml.put_cache(%{}) + assert Toml.fetch("work_dir") == :error + end +end From c3be05112d1efff8ddbe7213e3af2ef321e28ffb Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 20:26:03 +0000 Subject: [PATCH 03/47] feat(cfg): layered get_cfg/1 resolver (runtime > toml > default) --- lib/hyper/cfg.ex | 75 ++++++++++++++++++++++++++++++++ test/hyper/cfg/resolver_test.exs | 36 +++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 lib/hyper/cfg.ex create mode 100644 test/hyper/cfg/resolver_test.exs diff --git a/lib/hyper/cfg.ex b/lib/hyper/cfg.ex new file mode 100644 index 00000000..e08338ac --- /dev/null +++ b/lib/hyper/cfg.ex @@ -0,0 +1,75 @@ +defmodule Hyper.Cfg do + @moduledoc """ + The one place every Hyper configuration value is read. + + Configuration is layered, highest priority first: + + 1. `/etc/hyper/config.exs` — runtime app env, unprivileged node only. The + right place for secrets loaded at boot. + 2. `/etc/hyper/config.toml` — static, **shared with `hyper-suidhelper`**. + Anything that influences a root process lives here so the two sides can + never drift. + 3. Compile-time `config/config.exs` — performance fine-tuning. + 4. Built-in defaults. + + Each value names *which* of these layers it may be read from. Privileged tool + paths and the helper-shared `[jails]` table are `config.toml`-only, so the + unprivileged `config.exs` can never override a root-impacting path. + + Read values through the focused submodules — `Hyper.Cfg.Tools`, + `Hyper.Cfg.Dirs`, `Hyper.Cfg.Jails`, `Hyper.Cfg.Budget`, ... — never reach for + `Application.get_env` or `Hyper.Cfg.Toml` directly. + """ + + defmodule MissingError do + @moduledoc "Raised when a required config value is absent from every permitted source." + defexception [:message] + end + + @type source :: + {:runtime, atom() | {module(), atom()}} + | {:toml, String.t()} + | {:default, term()} + + @doc false + @spec get_cfg([source]) :: term() + def get_cfg(sources) when is_list(sources) do + case resolve(sources) do + {:ok, value} -> + value + + :error -> + raise MissingError, + message: + "required config value is not set in any permitted source: " <> + inspect(Keyword.delete(sources, :default)) + end + end + + @spec resolve([source]) :: {:ok, term()} | :error + defp resolve([]), do: :error + defp resolve([source | rest]) do + case from(source) do + {:ok, value} -> {:ok, value} + :error -> resolve(rest) + end + end + + @spec from(source) :: {:ok, term()} | :error + defp from({:default, value}), do: {:ok, value} + defp from({:toml, path}), do: Hyper.Cfg.Toml.fetch(path) + + defp from({:runtime, {mod, key}}) do + case Application.get_env(:hyper, mod) do + kw when is_list(kw) -> Keyword.fetch(kw, key) + _ -> :error + end + end + + defp from({:runtime, key}) when is_atom(key) do + case Application.get_env(:hyper, key) do + nil -> :error + value -> {:ok, value} + end + end +end diff --git a/test/hyper/cfg/resolver_test.exs b/test/hyper/cfg/resolver_test.exs new file mode 100644 index 00000000..0752ce54 --- /dev/null +++ b/test/hyper/cfg/resolver_test.exs @@ -0,0 +1,36 @@ +defmodule Hyper.Cfg.ResolverTest do + use ExUnit.Case, async: false + + import Hyper.Cfg, only: [get_cfg: 1] + + setup do + on_exit(fn -> Application.delete_env(:hyper, :__cfg_test) end) + end + + test "list order is priority: runtime wins over toml wins over default" do + Application.put_env(:hyper, :__cfg_test, "from_runtime") + + assert get_cfg(runtime: :__cfg_test, toml: "nope", default: "d") == "from_runtime" + end + + test "falls through absent runtime to the default" do + Application.delete_env(:hyper, :__cfg_test) + + assert get_cfg(runtime: :__cfg_test, default: "d") == "d" + end + + test "nested {mod, key} runtime source reads a keyword under the module env" do + Application.put_env(:hyper, __MODULE__, foo: "bar") + + assert get_cfg(runtime: {__MODULE__, :foo}, default: "d") == "bar" + assert get_cfg(runtime: {__MODULE__, :absent}, default: "d") == "d" + after + Application.delete_env(:hyper, __MODULE__) + end + + test "a required key (no default) with every source absent raises a named error" do + assert_raise Hyper.Cfg.MissingError, ~r/required/, fn -> + get_cfg(toml: "definitely.absent") + end + end +end From 26b7700a47d8975df1b1e554b9eefc687ea0099f Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 20:32:08 +0000 Subject: [PATCH 04/47] feat(cfg): Hyper.Cfg.Tools facade; define missing privileged tool paths --- lib/hyper/cfg/tools.ex | 64 +++++++++++++++++++++++++++++++ lib/hyper/img/oci_loader.ex | 8 ++-- lib/hyper/img/oci_loader/umoci.ex | 4 +- lib/hyper/suid_helper.ex | 4 +- lib/hyper/suid_helper/blockdev.ex | 4 +- lib/hyper/suid_helper/dmsetup.ex | 10 ++--- lib/hyper/suid_helper/losetup.ex | 8 ++-- test/hyper/cfg/tools_test.exs | 49 +++++++++++++++++++++++ 8 files changed, 132 insertions(+), 19 deletions(-) create mode 100644 lib/hyper/cfg/tools.ex create mode 100644 test/hyper/cfg/tools_test.exs diff --git a/lib/hyper/cfg/tools.ex b/lib/hyper/cfg/tools.ex new file mode 100644 index 00000000..3e6593f7 --- /dev/null +++ b/lib/hyper/cfg/tools.ex @@ -0,0 +1,64 @@ +defmodule Hyper.Cfg.Tools do + @moduledoc """ + Paths to every external binary Hyper runs, read from the `[tools]` table. + + The privileged tools (`firecracker`, `jailer`, `dmsetup`, `losetup`, + `blockdev`) are read **only** from `/etc/hyper/config.toml` — the file the + setuid helper also parses, so node and helper can never disagree on a + root-impacting path. The node-only tools (`skopeo`, `mke2fs`, `umoci`, + `suidhelper`) run unprivileged, so `/etc/hyper/config.exs` may override them + (e.g. a path from a secrets manager), then `config.toml`, then the default. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @doc "Firecracker binary. Required — raises if `[tools] firecracker` is unset." + @spec firecracker :: Path.t() + def firecracker, do: get_cfg(toml: "tools.firecracker") + + @doc "Non-raising `firecracker/0`." + @spec firecracker_configured :: {:ok, Path.t()} | :error + def firecracker_configured, do: Hyper.Cfg.Toml.fetch("tools.firecracker") + + @doc "Jailer binary. Required — raises if `[tools] jailer` is unset." + @spec jailer :: Path.t() + def jailer, do: get_cfg(toml: "tools.jailer") + + @doc "Non-raising `jailer/0`." + @spec jailer_configured :: {:ok, Path.t()} | :error + def jailer_configured, do: Hyper.Cfg.Toml.fetch("tools.jailer") + + @doc "dmsetup binary (privileged, config.toml-only)." + @spec dmsetup :: Path.t() + def dmsetup, do: get_cfg(toml: "tools.dmsetup", default: "/usr/sbin/dmsetup") + + @doc "losetup binary (privileged, config.toml-only)." + @spec losetup :: Path.t() + def losetup, do: get_cfg(toml: "tools.losetup", default: "/usr/sbin/losetup") + + @doc "blockdev binary (privileged, config.toml-only)." + @spec blockdev :: Path.t() + def blockdev, do: get_cfg(toml: "tools.blockdev", default: "/usr/sbin/blockdev") + + @doc "skopeo binary (node tool). config.exs > config.toml > `skopeo` on PATH." + @spec skopeo :: String.t() + def skopeo, do: get_cfg(runtime: {__MODULE__, :skopeo}, toml: "tools.skopeo", default: "skopeo") + + @doc "mke2fs binary (node tool). config.exs > config.toml > `mke2fs` on PATH." + @spec mke2fs :: String.t() + def mke2fs, do: get_cfg(runtime: {__MODULE__, :mke2fs}, toml: "tools.mke2fs", default: "mke2fs") + + @doc "umoci binary (node tool), or `nil` to let Hyper download a pinned release." + @spec umoci :: String.t() | nil + def umoci, do: get_cfg(runtime: {__MODULE__, :umoci}, toml: "tools.umoci", default: nil) + + @doc "setuid device helper (node tool). config.exs > config.toml > install default." + @spec suidhelper :: String.t() + def suidhelper, + do: + get_cfg( + runtime: {__MODULE__, :suidhelper}, + toml: "tools.suidhelper", + default: "/usr/local/bin/hyper-suidhelper" + ) +end diff --git a/lib/hyper/img/oci_loader.ex b/lib/hyper/img/oci_loader.ex index a261c6f6..07be6334 100644 --- a/lib/hyper/img/oci_loader.ex +++ b/lib/hyper/img/oci_loader.ex @@ -81,9 +81,9 @@ defmodule Hyper.Img.OciLoader do def test_system do with {:ok, _arch} <- Sys.Arch.current() do tools = [ - {"skopeo", Config.skopeo_path()}, + {"skopeo", Hyper.Cfg.Tools.skopeo()}, {"umoci", Umoci.bin()}, - {"mke2fs", Config.mke2fs_path()} + {"mke2fs", Hyper.Cfg.Tools.mke2fs()} ] missing = for {name, path} <- tools, System.find_executable(path) == nil, do: name @@ -137,7 +137,7 @@ defmodule Hyper.Img.OciLoader do bundle = Path.join(tmp, "bundle") skopeo = - cmd(Config.skopeo_path(), [ + cmd(Hyper.Cfg.Tools.skopeo(), [ "copy", "--override-os", "linux", @@ -192,7 +192,7 @@ defmodule Hyper.Img.OciLoader do ["-t", "ext4", "-F", "-q", "-N", to_string(inodes), "-d", rootfs, staged] ++ ["#{Information.as_mib(size)}M"] - case tag(cmd(Config.mke2fs_path(), args), :mke2fs) do + case tag(cmd(Hyper.Cfg.Tools.mke2fs(), args), :mke2fs) do :ok -> {:ok, staged} diff --git a/lib/hyper/img/oci_loader/umoci.ex b/lib/hyper/img/oci_loader/umoci.ex index dae13585..5cd11bbf 100644 --- a/lib/hyper/img/oci_loader/umoci.ex +++ b/lib/hyper/img/oci_loader/umoci.ex @@ -40,7 +40,7 @@ defmodule Hyper.Img.OciLoader.Umoci do """ @spec ensure_installed() :: :ok | {:error, term()} def ensure_installed do - if Config.umoci_path() != nil do + if Hyper.Cfg.Tools.umoci() != nil do :ok else with {:ok, arch} <- Sys.Arch.current() do @@ -57,7 +57,7 @@ defmodule Hyper.Img.OciLoader.Umoci do """ @spec bin() :: Path.t() def bin do - case Config.umoci_path() do + case Hyper.Cfg.Tools.umoci() do nil -> {:ok, arch} = Sys.Arch.current() default_path(arch) diff --git a/lib/hyper/suid_helper.ex b/lib/hyper/suid_helper.ex index 690f004a..1a11992a 100644 --- a/lib/hyper/suid_helper.ex +++ b/lib/hyper/suid_helper.ex @@ -30,7 +30,7 @@ defmodule Hyper.SuidHelper do @spec exec([String.t()]) :: {:ok, map()} | {:error, err()} @decorate with_span("Hyper.SuidHelper.exec", include: [:argv]) def exec(argv) do - case System.cmd(Hyper.Config.suid_helper(), argv, stderr_to_stdout: true) do + case System.cmd(Hyper.Cfg.Tools.suidhelper(), argv, stderr_to_stdout: true) do {out, 0} -> {:ok, Jason.decode!(out)} {out, code} -> {:error, {code, String.trim(out)}} end @@ -91,7 +91,7 @@ defmodule Hyper.SuidHelper do @spec helper_present() :: :ok | {:error, :suid_helper_not_found} defp helper_present do - if System.find_executable(Hyper.Config.suid_helper()), + if System.find_executable(Hyper.Cfg.Tools.suidhelper()), do: :ok, else: {:error, :suid_helper_not_found} end diff --git a/lib/hyper/suid_helper/blockdev.ex b/lib/hyper/suid_helper/blockdev.ex index 9f675ea5..be9bc39e 100644 --- a/lib/hyper/suid_helper/blockdev.ex +++ b/lib/hyper/suid_helper/blockdev.ex @@ -11,7 +11,7 @@ defmodule Hyper.SuidHelper.Blockdev do @spec device_sectors(Path.t()) :: {:ok, pos_integer()} | {:error, err()} @decorate with_span("Hyper.SuidHelper.Blockdev.device_sectors", include: [:path]) def device_sectors(path) do - case SuidHelper.exec(["blockdev", "--bin", Hyper.Config.blockdev_path(), "--getsz", path]) do + case SuidHelper.exec(["blockdev", "--bin", Hyper.Cfg.Tools.blockdev(), "--getsz", path]) do {:ok, %{"sectors" => n}} -> {:ok, n} {:error, _} = err -> err end @@ -21,7 +21,7 @@ defmodule Hyper.SuidHelper.Blockdev do @spec test_system() :: :ok | {:error, :blockdev_not_found} @decorate with_span("Hyper.SuidHelper.Blockdev.test_system") def test_system do - if System.find_executable(Hyper.Config.blockdev_path()), + if System.find_executable(Hyper.Cfg.Tools.blockdev()), do: :ok, else: {:error, :blockdev_not_found} end diff --git a/lib/hyper/suid_helper/dmsetup.ex b/lib/hyper/suid_helper/dmsetup.ex index ef670634..935ef84f 100644 --- a/lib/hyper/suid_helper/dmsetup.ex +++ b/lib/hyper/suid_helper/dmsetup.ex @@ -62,7 +62,7 @@ defmodule Hyper.SuidHelper.Dmsetup do case SuidHelper.exec([ "dmsetup", "--bin", - Hyper.Config.dmsetup_path(), + Hyper.Cfg.Tools.dmsetup(), "remove", "--retry", name @@ -77,7 +77,7 @@ defmodule Hyper.SuidHelper.Dmsetup do @decorate with_span("Hyper.SuidHelper.Dmsetup.message", include: [:name, :message]) def message(name, message) do argv = - ["dmsetup", "--bin", Hyper.Config.dmsetup_path(), "message", name, "--message", message] + ["dmsetup", "--bin", Hyper.Cfg.Tools.dmsetup(), "message", name, "--message", message] case SuidHelper.exec(argv) do {:ok, _} -> :ok @@ -92,7 +92,7 @@ defmodule Hyper.SuidHelper.Dmsetup do @spec test_system() :: :ok | {:error, term()} @decorate with_span("Hyper.SuidHelper.Dmsetup.test_system") def test_system do - if System.find_executable(Hyper.Config.dmsetup_path()), + if System.find_executable(Hyper.Cfg.Tools.dmsetup()), do: test_targets(), else: {:error, :dmsetup_not_found} end @@ -101,7 +101,7 @@ defmodule Hyper.SuidHelper.Dmsetup do @spec test_targets() :: :ok | {:error, term()} @decorate with_span("Hyper.SuidHelper.Dmsetup.test_targets") def test_targets do - case System.cmd(Hyper.Config.dmsetup_path(), ["targets"], stderr_to_stdout: true) do + case System.cmd(Hyper.Cfg.Tools.dmsetup(), ["targets"], stderr_to_stdout: true) do {out, 0} -> have = parse_targets(out) missing = Enum.reject(@required_targets, &MapSet.member?(have, &1)) @@ -146,7 +146,7 @@ defmodule Hyper.SuidHelper.Dmsetup do @spec create(String.t(), String.t(), [String.t()]) :: {:ok, Path.t()} | {:error, err()} defp create(name, table, flags) do argv = - ["dmsetup", "--bin", Hyper.Config.dmsetup_path(), "create", name] ++ + ["dmsetup", "--bin", Hyper.Cfg.Tools.dmsetup(), "create", name] ++ flags ++ ["--table", table] case SuidHelper.exec(argv) do diff --git a/lib/hyper/suid_helper/losetup.ex b/lib/hyper/suid_helper/losetup.ex index d825b731..9ee9191a 100644 --- a/lib/hyper/suid_helper/losetup.ex +++ b/lib/hyper/suid_helper/losetup.ex @@ -11,7 +11,7 @@ defmodule Hyper.SuidHelper.Losetup do @spec attach_ro(Path.t()) :: {:ok, Path.t()} | {:error, err()} @decorate with_span("Hyper.SuidHelper.Losetup.attach_ro", include: [:path]) def attach_ro(path) do - case SuidHelper.exec(["losetup", "--bin", Hyper.Config.losetup_path(), "attach", path]) do + case SuidHelper.exec(["losetup", "--bin", Hyper.Cfg.Tools.losetup(), "attach", path]) do {:ok, %{"device" => dev}} -> {:ok, dev} {:error, _} = err -> err end @@ -24,7 +24,7 @@ defmodule Hyper.SuidHelper.Losetup do case SuidHelper.exec([ "losetup", "--bin", - Hyper.Config.losetup_path(), + Hyper.Cfg.Tools.losetup(), "attach", "--rw", path @@ -38,7 +38,7 @@ defmodule Hyper.SuidHelper.Losetup do @spec detach(Path.t()) :: :ok | {:error, err()} @decorate with_span("Hyper.SuidHelper.Losetup.detach", include: [:dev]) def detach(dev) do - case SuidHelper.exec(["losetup", "--bin", Hyper.Config.losetup_path(), "detach", dev]) do + case SuidHelper.exec(["losetup", "--bin", Hyper.Cfg.Tools.losetup(), "detach", dev]) do {:ok, _} -> :ok {:error, _} = err -> err end @@ -48,7 +48,7 @@ defmodule Hyper.SuidHelper.Losetup do @spec test_system() :: :ok | {:error, :losetup_not_found} @decorate with_span("Hyper.SuidHelper.Losetup.test_system") def test_system do - if System.find_executable(Hyper.Config.losetup_path()), + if System.find_executable(Hyper.Cfg.Tools.losetup()), do: :ok, else: {:error, :losetup_not_found} end diff --git a/test/hyper/cfg/tools_test.exs b/test/hyper/cfg/tools_test.exs new file mode 100644 index 00000000..77874593 --- /dev/null +++ b/test/hyper/cfg/tools_test.exs @@ -0,0 +1,49 @@ +defmodule Hyper.Cfg.ToolsTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.Tools + alias Hyper.Cfg.Toml + + setup do + # Hermetic: empty TOML cache so we assert built-in defaults, not the + # ambient /etc/hyper/config.toml. Restore the real cache afterward. + Toml.put_cache(%{}) + on_exit(fn -> Toml.reload() end) + :ok + end + + test "privileged tools with no config.toml fall back to their sbin defaults" do + assert Tools.dmsetup() == "/usr/sbin/dmsetup" + assert Tools.losetup() == "/usr/sbin/losetup" + assert Tools.blockdev() == "/usr/sbin/blockdev" + end + + test "privileged tool paths come only from the [tools] table" do + Toml.put_cache(%{"tools" => %{"dmsetup" => "/custom/dmsetup"}}) + assert Tools.dmsetup() == "/custom/dmsetup" + end + + test "node tools default to bare PATH names / known install path" do + assert Tools.skopeo() == "skopeo" + assert Tools.mke2fs() == "mke2fs" + assert Tools.suidhelper() == "/usr/local/bin/hyper-suidhelper" + assert Tools.umoci() == nil + end + + test "node tools: config.exs (runtime) overrides config.toml" do + Toml.put_cache(%{"tools" => %{"skopeo" => "/from/toml"}}) + assert Tools.skopeo() == "/from/toml" + + Application.put_env(:hyper, Tools, skopeo: "/from/exs") + assert Tools.skopeo() == "/from/exs" + after + Application.delete_env(:hyper, Tools) + end + + test "required tools raise (non-raising form returns :error) when unset" do + assert Tools.firecracker_configured() == :error + assert Tools.jailer_configured() == :error + assert_raise Hyper.Cfg.MissingError, fn -> Tools.firecracker() end + assert_raise Hyper.Cfg.MissingError, fn -> Tools.jailer() end + end +end From 612d1038f270dc3cd3a2b5d8f549f87bf432e6d4 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 20:41:05 +0000 Subject: [PATCH 05/47] feat(cfg): Hyper.Cfg.Dirs facade; define firecracker_install_dir Creates Hyper.Cfg.Dirs with work_dir/0 (config.toml-only, default /srv/hyper) and all 8 derived directory accessors. Migrates every call-site away from Hyper.Config.*; removes now-unused aliases in img.ex, oci_loader.ex, umoci.ex. Resolves the last undefined-accessor warning: firecracker_install_dir/0. --- lib/hyper/cfg/dirs.ex | 47 ++++++++++++++++++++ lib/hyper/img.ex | 7 ++- lib/hyper/img/oci_loader.ex | 5 +-- lib/hyper/img/oci_loader/umoci.ex | 8 ++-- lib/hyper/node.ex | 4 +- lib/hyper/node/fire_vmm/jailer.ex | 8 ++-- lib/hyper/node/fire_vmm/provider.ex | 4 +- lib/hyper/node/fire_vmm/vm_linux/provider.ex | 4 +- lib/hyper/node/img/thin_pool.ex | 4 +- lib/hyper/node/layer/repo.ex | 6 +-- test/hyper/cfg/dirs_test.exs | 37 +++++++++++++++ 11 files changed, 107 insertions(+), 27 deletions(-) create mode 100644 lib/hyper/cfg/dirs.ex create mode 100644 test/hyper/cfg/dirs_test.exs diff --git a/lib/hyper/cfg/dirs.ex b/lib/hyper/cfg/dirs.ex new file mode 100644 index 00000000..4010e50d --- /dev/null +++ b/lib/hyper/cfg/dirs.ex @@ -0,0 +1,47 @@ +defmodule Hyper.Cfg.Dirs do + @moduledoc """ + The node's work directory and every directory derived from it. + + `work_dir` is the single configurable root (`config.toml`-only, shared with the + setuid helper); everything else is a fixed sub-path so the node and helper + agree on layout without a second key to keep in sync. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @doc "Root work directory for this node. config.toml `work_dir`, default `/srv/hyper`." + @spec work_dir :: Path.t() + def work_dir, do: get_cfg(toml: "work_dir", default: "/srv/hyper") + + @doc "Read-only image layer store (`/layers`)." + @spec layer_dir :: Path.t() + def layer_dir, do: Path.join(work_dir(), "layers") + + @doc "Per-VM control/gRPC sockets (`/socks`)." + @spec socket_dir :: Path.t() + def socket_dir, do: Path.join(work_dir(), "socks") + + @doc "Per-VM copy-on-write writable layers (`/scratch`)." + @spec scratch_dir :: Path.t() + def scratch_dir, do: Path.join(work_dir(), "scratch") + + @doc "Per-VM chroot directories (`/jails`)." + @spec chroot_base :: Path.t() + def chroot_base, do: Path.join(work_dir(), "jails") + + @doc "Node-downloaded binaries (`/redist`)." + @spec redist_dir :: Path.t() + def redist_dir, do: Path.join(work_dir(), "redist") + + @doc "Where guest kernels install (`/redist/vmlinux`)." + @spec vmlinux_install_dir :: Path.t() + def vmlinux_install_dir, do: Path.join(redist_dir(), "vmlinux") + + @doc "Where the default umoci installs (`/redist/umoci`)." + @spec umoci_install_dir :: Path.t() + def umoci_install_dir, do: Path.join(redist_dir(), "umoci") + + @doc "Where a node-downloaded firecracker installs (`/redist/firecracker`)." + @spec firecracker_install_dir :: Path.t() + def firecracker_install_dir, do: Path.join(redist_dir(), "firecracker") +end diff --git a/lib/hyper/img.ex b/lib/hyper/img.ex index f793d168..537dc824 100644 --- a/lib/hyper/img.ex +++ b/lib/hyper/img.ex @@ -6,7 +6,7 @@ defmodule Hyper.Img do `create/2` ingests a prepared image file -- e.g. the ext4 rootfs produced by `Hyper.Img.OciLoader` -- into the shared media store and the image database. It content-addresses the file (sha256 of its bytes = the image id), publishes it - into `Hyper.Config.layer_dir/0` at `layer_.img`, then records it as a + into `Hyper.Cfg.Dirs.layer_dir/0` at `layer_.img`, then records it as a one-layer base image (`blobs` + `images` + `image_layers`). Producers of image files stay decoupled from the store and DB: they hand a path to `create/2`. """ @@ -15,7 +15,6 @@ defmodule Hyper.Img do require Logger - alias Hyper.Config alias Hyper.Img.Db.{Blob, Image, ImageLayer, Repo} @type id :: String.t() @@ -85,7 +84,7 @@ defmodule Hyper.Img do @spec publish(Path.t(), id()) :: {:ok, Path.t(), :created | :reused} | {:error, term()} @decorate with_span("Hyper.Img.publish", include: [:id]) defp publish(src, id) do - File.mkdir_p!(Config.layer_dir()) + File.mkdir_p!(Hyper.Cfg.Dirs.layer_dir()) final = final_path(id) if File.exists?(final) do @@ -125,7 +124,7 @@ defmodule Hyper.Img do end @spec final_path(id()) :: Path.t() - defp final_path(id), do: Path.join(Config.layer_dir(), "layer_#{id}.img") + defp final_path(id), do: Path.join(Hyper.Cfg.Dirs.layer_dir(), "layer_#{id}.img") # Record the base image: one blob, one image (id == blob id), one layer at # position 0. All upserts are idempotent so a re-publish of the same bytes is a diff --git a/lib/hyper/img/oci_loader.ex b/lib/hyper/img/oci_loader.ex index 07be6334..ab543614 100644 --- a/lib/hyper/img/oci_loader.ex +++ b/lib/hyper/img/oci_loader.ex @@ -20,7 +20,6 @@ defmodule Hyper.Img.OciLoader do use Unit.Operators - alias Hyper.Config alias Hyper.Img.OciLoader.Umoci alias Unit.Information @@ -185,8 +184,8 @@ defmodule Hyper.Img.OciLoader do {:ok, Path.t()} | {:error, term()} defp build_ext4(rootfs, {size, inodes}) do Logger.debug("oci: building #{Information.as_mib(size)} MiB ext4 rootfs (#{inodes} inodes)") - File.mkdir_p!(Config.layer_dir()) - staged = Path.join(Config.layer_dir(), ".incoming-#{System.unique_integer([:positive])}.img") + File.mkdir_p!(Hyper.Cfg.Dirs.layer_dir()) + staged = Path.join(Hyper.Cfg.Dirs.layer_dir(), ".incoming-#{System.unique_integer([:positive])}.img") args = ["-t", "ext4", "-F", "-q", "-N", to_string(inodes), "-d", rootfs, staged] ++ diff --git a/lib/hyper/img/oci_loader/umoci.ex b/lib/hyper/img/oci_loader/umoci.ex index 5cd11bbf..3f7d3b6d 100644 --- a/lib/hyper/img/oci_loader/umoci.ex +++ b/lib/hyper/img/oci_loader/umoci.ex @@ -6,14 +6,12 @@ defmodule Hyper.Img.OciLoader.Umoci do Two sources, in priority order (mirrors `Hyper.Node.Vmlinux`): 1. An operator-configured path via `[tools] umoci` in - `/etc/hyper/config.toml` (`Hyper.Config.umoci_path/0`). If set, it wins + `/etc/hyper/config.toml` (`Hyper.Cfg.Tools.umoci/0`). If set, it wins and is never downloaded. 2. Otherwise the pinned static binary downloaded by `ensure_installed/0` - into `Hyper.Config.umoci_install_dir/0` (`/redist/umoci`). + into `Hyper.Cfg.Dirs.umoci_install_dir/0` (`/redist/umoci`). """ - alias Hyper.Config - require Logger # Pinned umoci release per architecture: the static binary's filename, its @@ -69,7 +67,7 @@ defmodule Hyper.Img.OciLoader.Umoci do @spec default_path(Sys.Arch.t()) :: Path.t() defp default_path(arch) do - Path.join(Config.umoci_install_dir(), Map.fetch!(@downloads, arch).asset) + Path.join(Hyper.Cfg.Dirs.umoci_install_dir(), Map.fetch!(@downloads, arch).asset) end @spec install(Sys.Arch.t(), Path.t()) :: :ok | {:error, term()} diff --git a/lib/hyper/node.ex b/lib/hyper/node.ex index 6282eaa7..6591cb22 100644 --- a/lib/hyper/node.ex +++ b/lib/hyper/node.ex @@ -160,10 +160,10 @@ defmodule Hyper.Node do @spec check_helper_base(Path.t()) :: :ok | {:error, {:suid_helper_base_mismatch, Path.t(), Path.t()}} defp check_helper_base(base) do - if base == Hyper.Config.work_dir() do + if base == Hyper.Cfg.Dirs.work_dir() do :ok else - {:error, {:suid_helper_base_mismatch, base, Hyper.Config.work_dir()}} + {:error, {:suid_helper_base_mismatch, base, Hyper.Cfg.Dirs.work_dir()}} end end diff --git a/lib/hyper/node/fire_vmm/jailer.ex b/lib/hyper/node/fire_vmm/jailer.ex index 6dd1b800..332acf0b 100644 --- a/lib/hyper/node/fire_vmm/jailer.ex +++ b/lib/hyper/node/fire_vmm/jailer.ex @@ -79,7 +79,7 @@ defmodule Hyper.Node.FireVMM.Jailer do end defp chroot_writable do - case Sys.Posix.ensure_writable_dir(Config.chroot_base()) do + case Sys.Posix.ensure_writable_dir(Hyper.Cfg.Dirs.chroot_base()) do {:ok} -> :ok {:error, reason} -> {:error, {:chroot_base_unavailable, reason}} end @@ -104,7 +104,7 @@ defmodule Hyper.Node.FireVMM.Jailer do "--gid", to_string(opts.gid), "--chroot-base-dir", - Hyper.Config.chroot_base(), + Hyper.Cfg.Dirs.chroot_base(), "--cgroup-version", "2", "--parent-cgroup", @@ -129,7 +129,7 @@ defmodule Hyper.Node.FireVMM.Jailer do @doc "Host path of the VM's per-VM jail dir (`//`)." @spec chroot_dir(Hyper.Vm.id()) :: Path.t() def chroot_dir(id) do - Path.join([Hyper.Config.chroot_base(), exec_name(), id]) + Path.join([Hyper.Cfg.Dirs.chroot_base(), exec_name(), id]) end @doc "Host path of the VM's chroot root (`///root`)." @@ -158,7 +158,7 @@ defmodule Hyper.Node.FireVMM.Jailer do @spec host_socket(Hyper.Vm.id()) :: Path.t() def host_socket(id) do Path.join([ - Hyper.Config.chroot_base(), + Hyper.Cfg.Dirs.chroot_base(), exec_name(), id, "root", diff --git a/lib/hyper/node/fire_vmm/provider.ex b/lib/hyper/node/fire_vmm/provider.ex index e50aee16..0730c588 100644 --- a/lib/hyper/node/fire_vmm/provider.ex +++ b/lib/hyper/node/fire_vmm/provider.ex @@ -1,7 +1,7 @@ defmodule Hyper.Node.FireVMM.Provider do @moduledoc """ Installs the firecracker release for the current architecture into - `Hyper.Config.firecracker_install_dir/0` (`/redist/firecracker`). + `Hyper.Cfg.Dirs.firecracker_install_dir/0` (`/redist/firecracker`). `ensure_installed/0` is idempotent: if the binaries are already present and executable it returns `:ok` without touching the network. Otherwise it fetches @@ -88,5 +88,5 @@ defmodule Hyper.Node.FireVMM.Provider do Path.join(install_dir(), Map.fetch!(dl, key)) end - defp install_dir, do: Hyper.Config.firecracker_install_dir() + defp install_dir, do: Hyper.Cfg.Dirs.firecracker_install_dir() end diff --git a/lib/hyper/node/fire_vmm/vm_linux/provider.ex b/lib/hyper/node/fire_vmm/vm_linux/provider.ex index 0b8f7682..b01daf84 100644 --- a/lib/hyper/node/fire_vmm/vm_linux/provider.ex +++ b/lib/hyper/node/fire_vmm/vm_linux/provider.ex @@ -1,7 +1,7 @@ defmodule Hyper.Node.FireVMM.VmLinux.Provider do @moduledoc """ Installs the guest-kernel (vmlinux) images for the current architecture into - `Hyper.Config.vmlinux_install_dir/0` (`/redist/vmlinux`). + `Hyper.Cfg.Dirs.vmlinux_install_dir/0` (`/redist/vmlinux`). The available kernels and their SHA-256 sums come from the statically-embedded `Hyper.Node.FireVMM.VmLinux.Manifest`. `ensure_installed/0` installs *every* @@ -89,5 +89,5 @@ defmodule Hyper.Node.FireVMM.VmLinux.Provider do defp build_path(dir, build), do: Path.join(dir, build.asset) - defp install_dir, do: Hyper.Config.vmlinux_install_dir() + defp install_dir, do: Hyper.Cfg.Dirs.vmlinux_install_dir() end diff --git a/lib/hyper/node/img/thin_pool.ex b/lib/hyper/node/img/thin_pool.ex index f3ec3e19..bda11d7e 100644 --- a/lib/hyper/node/img/thin_pool.ex +++ b/lib/hyper/node/img/thin_pool.ex @@ -50,7 +50,7 @@ defmodule Hyper.Node.Img.ThinPool do def init(_opts) do Process.flag(:trap_exit, true) - with :ok <- File.mkdir_p(Hyper.Config.scratch_dir()), + with :ok <- File.mkdir_p(Hyper.Cfg.Dirs.scratch_dir()), {:ok, meta} <- ensure_backing(@meta_file, ImgConfig.thin_pool_meta_size()), {:ok, data} <- ensure_backing(@data_file, ImgConfig.thin_pool_data_size()), :ok <- zero_metadata(meta), @@ -114,7 +114,7 @@ defmodule Hyper.Node.Img.ThinPool do # Create a sparse file of `size` if absent; reuse it if already present. @spec ensure_backing(String.t(), Information.t()) :: {:ok, Path.t()} | {:error, term()} defp ensure_backing(file, size) do - path = Path.join(Hyper.Config.scratch_dir(), file) + path = Path.join(Hyper.Cfg.Dirs.scratch_dir(), file) case File.open(path, [:write, :read]) do {:ok, io} -> diff --git a/lib/hyper/node/layer/repo.ex b/lib/hyper/node/layer/repo.ex index 6604a08c..f9cf0be5 100644 --- a/lib/hyper/node/layer/repo.ex +++ b/lib/hyper/node/layer/repo.ex @@ -15,8 +15,8 @@ defmodule Hyper.Node.Layer.Repo do @decorate with_span("Hyper.Node.Layer.Repo.test_system") def test_system do cond do - not File.exists?(Hyper.Config.layer_dir()) -> {:error, :layer_dir_not_found} - not File.dir?(Hyper.Config.layer_dir()) -> {:error, :layer_dir_not_dir} + not File.exists?(Hyper.Cfg.Dirs.layer_dir()) -> {:error, :layer_dir_not_found} + not File.dir?(Hyper.Cfg.Dirs.layer_dir()) -> {:error, :layer_dir_not_dir} true -> :ok end end @@ -25,7 +25,7 @@ defmodule Hyper.Node.Layer.Repo do @spec find_layer(Hyper.Layer.id()) :: {:ok, Path.t()} | {:error, File.posix()} @decorate with_span("Hyper.Node.Layer.Repo.find_layer") def find_layer(id) do - path = Path.join([Hyper.Config.layer_dir(), layer_basename(id)]) + path = Path.join([Hyper.Cfg.Dirs.layer_dir(), layer_basename(id)]) case File.stat(path) do {:ok, _stat} -> {:ok, path} diff --git a/test/hyper/cfg/dirs_test.exs b/test/hyper/cfg/dirs_test.exs new file mode 100644 index 00000000..394fddb2 --- /dev/null +++ b/test/hyper/cfg/dirs_test.exs @@ -0,0 +1,37 @@ +defmodule Hyper.Cfg.DirsTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.Dirs + alias Hyper.Cfg.Toml + + setup do + Toml.put_cache(%{}) + on_exit(fn -> Toml.reload() end) + :ok + end + + test "work_dir defaults to /srv/hyper and every dir derives from it" do + root = Dirs.work_dir() + assert root == "/srv/hyper" + + assert Dirs.layer_dir() == Path.join(root, "layers") + assert Dirs.socket_dir() == Path.join(root, "socks") + assert Dirs.scratch_dir() == Path.join(root, "scratch") + assert Dirs.chroot_base() == Path.join(root, "jails") + assert Dirs.redist_dir() == Path.join(root, "redist") + end + + test "redistributable install dirs nest under redist" do + redist = Dirs.redist_dir() + assert Dirs.vmlinux_install_dir() == Path.join(redist, "vmlinux") + assert Dirs.umoci_install_dir() == Path.join(redist, "umoci") + assert Dirs.firecracker_install_dir() == Path.join(redist, "firecracker") + end + + test "work_dir follows the config.toml value and dirs re-derive" do + Toml.put_cache(%{"work_dir" => "/data/hyper"}) + assert Dirs.work_dir() == "/data/hyper" + assert Dirs.layer_dir() == "/data/hyper/layers" + assert Dirs.firecracker_install_dir() == "/data/hyper/redist/firecracker" + end +end From 49f204dec983da87c7b8cf04cb30324b15748aa0 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 20:45:07 +0000 Subject: [PATCH 06/47] feat(cfg): Hyper.Cfg.Jails facade (cgroup, uid_gid_range) --- lib/hyper/cfg/jails.ex | 28 ++++++++++++++++++++++++++++ lib/hyper/node/fire_vmm/jailer.ex | 8 +++----- lib/hyper/node/users.ex | 4 ++-- test/hyper/cfg/jails_test.exs | 29 +++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 lib/hyper/cfg/jails.ex create mode 100644 test/hyper/cfg/jails_test.exs diff --git a/lib/hyper/cfg/jails.ex b/lib/hyper/cfg/jails.ex new file mode 100644 index 00000000..df2056a3 --- /dev/null +++ b/lib/hyper/cfg/jails.ex @@ -0,0 +1,28 @@ +defmodule Hyper.Cfg.Jails do + @moduledoc """ + VM confinement settings from the `[jails]` table — `config.toml`-only because + the setuid helper enforces the same `uid_gid_range` it reads from this file. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @default_range {900_000, 999_999} + + @doc "Parent cgroup for every VM cgroup. `[jails] cgroup`, default `\"hyper\"`." + @spec cgroup :: String.t() + def cgroup, do: get_cfg(toml: "jails.cgroup", default: "hyper") + + @doc "UID/GID allocation band each VM jail draws from. `[jails] uid_gid_range`." + @spec uid_gid_range :: {integer(), integer()} + def uid_gid_range do + case Hyper.Cfg.Toml.fetch("jails.uid_gid_range") do + {:ok, v} -> range_from(v) + :error -> @default_range + end + end + + @doc false + @spec range_from(term()) :: {integer(), integer()} + def range_from([min, max]) when is_integer(min) and is_integer(max), do: {min, max} + def range_from(_), do: @default_range +end diff --git a/lib/hyper/node/fire_vmm/jailer.ex b/lib/hyper/node/fire_vmm/jailer.ex index 332acf0b..381f9d23 100644 --- a/lib/hyper/node/fire_vmm/jailer.ex +++ b/lib/hyper/node/fire_vmm/jailer.ex @@ -36,8 +36,6 @@ defmodule Hyper.Node.FireVMM.Jailer do first failure. """ - alias Hyper.Config - @doc "Run every pre-requisite check, halting at the first failure." @spec run() :: :ok | {:error, term()} def run do @@ -63,7 +61,7 @@ defmodule Hyper.Node.FireVMM.Jailer do end defp parent_cgroup_present do - if Sys.Linux.Cgroup.V2.named_exists?(Config.parent_cgroup()), + if Sys.Linux.Cgroup.V2.named_exists?(Hyper.Cfg.Jails.cgroup()), do: :ok, else: {:error, :missing_parent_cgroup} end @@ -108,7 +106,7 @@ defmodule Hyper.Node.FireVMM.Jailer do "--cgroup-version", "2", "--parent-cgroup", - Hyper.Config.parent_cgroup() + Hyper.Cfg.Jails.cgroup() ] ++ cgroup_flags(opts.type) ++ ["--", "--api-sock", "/" <> @jail_socket] @@ -145,7 +143,7 @@ defmodule Hyper.Node.FireVMM.Jailer do """ @spec cgroup_dir(Hyper.Vm.id()) :: Path.t() def cgroup_dir(id) do - Path.join(["/sys/fs/cgroup", Hyper.Config.parent_cgroup(), exec_name(), id]) + Path.join(["/sys/fs/cgroup", Hyper.Cfg.Jails.cgroup(), exec_name(), id]) end @doc """ diff --git a/lib/hyper/node/users.ex b/lib/hyper/node/users.ex index 72777d1f..6643da79 100644 --- a/lib/hyper/node/users.ex +++ b/lib/hyper/node/users.ex @@ -77,7 +77,7 @@ defmodule Hyper.Node.Users do def test_system do alias Sys.Linux.Subid - {min, max} = Hyper.Config.uid_gid_range() + {min, max} = Hyper.Cfg.Jails.uid_gid_range() cond do passwd_conflicts(min, max) != [] -> @@ -99,7 +99,7 @@ defmodule Hyper.Node.Users do @impl true def init(_opts) do - {min, max} = Hyper.Config.uid_gid_range() + {min, max} = Hyper.Cfg.Jails.uid_gid_range() {:ok, %State{max: max, next: min}} end diff --git a/test/hyper/cfg/jails_test.exs b/test/hyper/cfg/jails_test.exs new file mode 100644 index 00000000..c9791156 --- /dev/null +++ b/test/hyper/cfg/jails_test.exs @@ -0,0 +1,29 @@ +defmodule Hyper.Cfg.JailsTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.Jails + alias Hyper.Cfg.Toml + + setup do + Toml.put_cache(%{}) + on_exit(fn -> Toml.reload() end) + :ok + end + + test "defaults match the helper's compiled-in defaults" do + assert Jails.cgroup() == "hyper" + assert Jails.uid_gid_range() == {900_000, 999_999} + end + + test "reads the [jails] table when present" do + Toml.put_cache(%{"jails" => %{"cgroup" => "fleet", "uid_gid_range" => [800_000, 899_999]}}) + assert Jails.cgroup() == "fleet" + assert Jails.uid_gid_range() == {800_000, 899_999} + end + + test "uid_gid_range parses a TOML [min, max] array into a tuple" do + assert Jails.range_from([800_000, 899_999]) == {800_000, 899_999} + assert Jails.range_from(nil) == {900_000, 999_999} + assert Jails.range_from("garbage") == {900_000, 999_999} + end +end From daf81c94587b4fd3a61918c3de31ee97fe217d26 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 20:48:17 +0000 Subject: [PATCH 07/47] feat(cfg): Hyper.Cfg.Vmlinux facade for the kernel map --- lib/hyper/cfg/vmlinux.ex | 14 ++++++++++++++ lib/hyper/node/vmlinux.ex | 6 +++--- test/hyper/cfg/vmlinux_test.exs | 13 +++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 lib/hyper/cfg/vmlinux.ex create mode 100644 test/hyper/cfg/vmlinux_test.exs diff --git a/lib/hyper/cfg/vmlinux.ex b/lib/hyper/cfg/vmlinux.ex new file mode 100644 index 00000000..3b4ee8be --- /dev/null +++ b/lib/hyper/cfg/vmlinux.ex @@ -0,0 +1,14 @@ +defmodule Hyper.Cfg.Vmlinux do + @moduledoc """ + Per-architecture guest-kernel image paths, set by the operator in + `config :hyper, vmlinux: %{arch => path}`. Node-only; no helper counterpart. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @doc "Operator-configured `%{arch => path}` kernel map, default `%{}`." + # Runtime read, not compile_env: an unset map would inline a literal `%{}`, + # which the type checker proves makes every Map.fetch/2 on it return :error. + @spec images :: %{optional(Sys.Arch.t()) => Path.t()} + def images, do: get_cfg(runtime: :vmlinux, default: %{}) +end diff --git a/lib/hyper/node/vmlinux.ex b/lib/hyper/node/vmlinux.ex index 92f7f688..8742a258 100644 --- a/lib/hyper/node/vmlinux.ex +++ b/lib/hyper/node/vmlinux.ex @@ -5,7 +5,7 @@ defmodule Hyper.Node.Vmlinux do Two sources, in priority order: 1. An operator-configured path for the node's architecture, via - `config :hyper, vmlinux: %{ => }` (see `Hyper.Config.vmlinux/0`). + `config :hyper, vmlinux: %{ => }` (see `Hyper.Cfg.Vmlinux.images/0`). If set, it wins - the operator can pin a custom kernel. 2. Otherwise, the default kernel downloaded by `Hyper.Node.FireVMM.VmLinux.Provider` (highest version for the arch). @@ -24,7 +24,7 @@ defmodule Hyper.Node.Vmlinux do """ @spec path(Sys.Arch.t()) :: Path.t() def path(arch) do - case Map.fetch(Hyper.Config.vmlinux(), arch) do + case Map.fetch(Hyper.Cfg.Vmlinux.images(), arch) do {:ok, path} -> path @@ -44,7 +44,7 @@ defmodule Hyper.Node.Vmlinux do @spec test_system() :: :ok | {:error, term()} def test_system do with {:ok, arch} <- Sys.Arch.current() do - case Map.fetch(Hyper.Config.vmlinux(), arch) do + case Map.fetch(Hyper.Cfg.Vmlinux.images(), arch) do {:ok, path} -> present(path) diff --git a/test/hyper/cfg/vmlinux_test.exs b/test/hyper/cfg/vmlinux_test.exs new file mode 100644 index 00000000..4dc0dac6 --- /dev/null +++ b/test/hyper/cfg/vmlinux_test.exs @@ -0,0 +1,13 @@ +defmodule Hyper.Cfg.VmlinuxTest do + use ExUnit.Case, async: false + + test "images/0 defaults to an empty map and reads config :hyper, :vmlinux" do + Application.delete_env(:hyper, :vmlinux) + assert Hyper.Cfg.Vmlinux.images() == %{} + + Application.put_env(:hyper, :vmlinux, %{x86_64: "/k/vmlinux"}) + assert Hyper.Cfg.Vmlinux.images() == %{x86_64: "/k/vmlinux"} + after + Application.delete_env(:hyper, :vmlinux) + end +end From 9983f9ebc5286a397f24e8bfded3dd84c4ce7441 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 20:51:13 +0000 Subject: [PATCH 08/47] refactor(cfg): remove Hyper.Config and dead config.exs keys Delete the now-redundant Hyper.Config module (all accessors have been re-homed under Hyper.Cfg.*). Migrate the two test files that still referenced it, and remove the three dead flat keys (cgroup_parent, uid_gid_range, layer_dir) from config/config.exs that were never read via Application.get_env. --- config/config.exs | 5 - lib/hyper/config.ex | 265 ------------------ test/hyper/img/oci_loader_test.exs | 4 +- .../node/fire_vmm/vm_linux/provider_test.exs | 4 +- 4 files changed, 4 insertions(+), 274 deletions(-) delete mode 100644 lib/hyper/config.ex diff --git a/config/config.exs b/config/config.exs index 7c831bce..cb0bd469 100644 --- a/config/config.exs +++ b/config/config.exs @@ -21,11 +21,6 @@ config :libcluster, ] ] -config :hyper, - cgroup_parent: "hyper", - uid_gid_range: {900_000, 999_999}, - layer_dir: "/srv/hyper/layers" - if config_env() == :test do config :opentelemetry, traces_exporter: :none # No cluster formation during tests. diff --git a/lib/hyper/config.ex b/lib/hyper/config.ex deleted file mode 100644 index b39ae96b..00000000 --- a/lib/hyper/config.ex +++ /dev/null @@ -1,265 +0,0 @@ -defmodule Hyper.Config do - @moduledoc """ - Host configuration. - - Everything shared with the setuid helper (`native/suidhelper`) is read from the - single source of truth, `/etc/hyper/config.toml`, at runtime — never duplicated - in `config :hyper`. The node and the helper parse the same file, so they cannot - drift: `work_dir`, the `[tools]` binary paths (`firecracker`, `jailer`, ...), - and the `[jails]` table (`cgroup`, `uid_gid_range`). The file is read once on first access - and cached in `:persistent_term`; an absent file (local dev / CI) yields the - same built-in defaults the helper compiles in, so both sides still agree. - - The node's own tools (`skopeo`, `umoci`, `mke2fs`, `suidhelper`) share that same - `[tools]` table — the helper simply ignores the keys it does not recognise, so - one table serves both. Only `vmlinux` and the cluster topology remain in - `config :hyper`. - """ - - # The shared config file, read by both this node and the setuid helper. Absent - # in local dev / CI, where the built-in defaults below are used instead. - @config_path "/etc/hyper/config.toml" - @dev_work_dir "/srv/hyper" - - # Defaults for the helper-shared values, kept in lockstep with the helper's - # `Config::default` (native/suidhelper/src/config.rs) so an absent config.toml - # makes the node and the helper agree out of the box. - @default_parent_cgroup "hyper" - @default_uid_gid_range {900_000, 999_999} - - # Defaults for the node-only `[tools]` binaries (skopeo/umoci/mke2fs/suidhelper). - # These bare keys live alongside the helper's own tools in the `[tools]` table; - # the helper ignores the keys it does not recognise, so the two sides share one - # table without colliding. - @default_skopeo "skopeo" - @default_mke2fs "mke2fs" - @default_suid_helper "/usr/local/bin/hyper-suidhelper" - - @doc """ - Root work directory for this node. All firecracker paths derive from it. - - Read from `#{@config_path}` (the single source of truth shared with the setuid - helper) the first time it is needed, then cached via `config_toml/0`. Falls back - to `#{@dev_work_dir}` when the file is absent (local dev / CI, where the helper - is not installed anyway). - """ - @spec work_dir :: Path.t() - def work_dir, do: Map.get(config_toml(), "work_dir", @dev_work_dir) - - @doc "Directory holding redistributable binaries downloaded by the node." - @spec redist_dir :: Path.t() - def redist_dir, do: Path.join(work_dir(), "redist") - - @doc """ - Absolute path to the firecracker binary, from the `[tools]` table in - `#{@config_path}`. Raises if absent — the operator must configure it; there is - no default. - - For the launch path only. Pre-launch checks should use `firecracker_bin_configured/0` - so a missing key returns a typed error rather than crashing. - """ - @spec firecracker_bin :: Path.t() - def firecracker_bin, do: fetch_tool!("firecracker") - - @doc """ - Non-raising form of `firecracker_bin/0`. Returns `{:ok, path}` when the - `[tools] firecracker` key is present in `#{@config_path}`, or `:error` otherwise. - """ - @spec firecracker_bin_configured :: {:ok, Path.t()} | :error - def firecracker_bin_configured, do: Map.fetch(tools(), "firecracker") - - @doc """ - Absolute path to the jailer binary, from the `[tools]` table in `#{@config_path}`. - Raises if absent — the operator must configure it; there is no default. - - For the launch path only. Pre-launch checks should use `jailer_bin_configured/0` - so a missing key returns a typed error rather than crashing. - """ - @spec jailer_bin :: Path.t() - def jailer_bin, do: fetch_tool!("jailer") - - @doc """ - Non-raising form of `jailer_bin/0`. Returns `{:ok, path}` when the - `[tools] jailer` key is present in `#{@config_path}`, or `:error` otherwise. - """ - @spec jailer_bin_configured :: {:ok, Path.t()} | :error - def jailer_bin_configured, do: Map.fetch(tools(), "jailer") - - # The `[tools]` table (binary paths shared with the helper), or `%{}` when the - # file or table is absent. - @spec tools :: map() - defp tools, do: Map.get(config_toml(), "tools", %{}) - - @spec fetch_tool!(String.t()) :: Path.t() - defp fetch_tool!(key) do - case Map.fetch(tools(), key) do - {:ok, path} -> - path - - :error -> - raise "#{@config_path}: `[tools] #{key}` is not set; " <> - "operator must configure it before starting the node" - end - end - - # A `[tools]` path with a built-in default: the configured string, or `default` - # when the key is absent (or set to a non-string, treated as unset). The - # `is_binary/1` guard pins the success type to `String.t()` so the public - # accessors stay precisely typed for Dialyzer rather than widening to `any()`. - @spec tool_path(String.t(), String.t()) :: String.t() - defp tool_path(key, default) do - case Map.get(tools(), key) do - path when is_binary(path) -> path - _ -> default - end - end - - # A `[tools]` path with no default: the configured string, or `nil` when unset. - @spec optional_tool_path(String.t()) :: String.t() | nil - defp optional_tool_path(key) do - case Map.get(tools(), key) do - path when is_binary(path) -> path - _ -> nil - end - end - - @spec config_toml :: map() - defp config_toml do - case :persistent_term.get({__MODULE__, :config_toml}, nil) do - nil -> - cfg = load_config_toml() - :persistent_term.put({__MODULE__, :config_toml}, cfg) - cfg - - cfg -> - cfg - end - end - - @spec load_config_toml :: map() - defp load_config_toml do - case File.read(@config_path) do - {:ok, body} -> Toml.decode!(body) - {:error, _} -> %{} - end - end - - @doc "Directory where `Hyper.Node.FireVMM.VmLinux.Provider` installs guest kernels." - @spec vmlinux_install_dir :: Path.t() - def vmlinux_install_dir, do: Path.join(redist_dir(), "vmlinux") - - @doc "Directory where `Hyper.Img.OciLoader.Umoci` installs the default umoci binary." - @spec umoci_install_dir :: Path.t() - def umoci_install_dir, do: Path.join(redist_dir(), "umoci") - - @doc """ - Path to the directory where all VM chroot's are created (`/jails`). - - If it does not exist, `Hyper.Node` will attempt to create one. - """ - @spec chroot_base :: Path.t() - def chroot_base, do: Path.join(work_dir(), "jails") - - @doc """ - Name of the parent cgroup used as a supervision cgroup for all VMs. Read from - `[jails] cgroup` in `#{@config_path}` (shared with the helper), default `"hyper"`. - """ - @spec parent_cgroup :: String.t() - def parent_cgroup, do: Map.get(jails(), "cgroup", @default_parent_cgroup) - - # The `[jails]` table (VM placement/confinement, shared with the helper), or - # `%{}` when the file or table is absent. - @spec jails :: map() - defp jails, do: Map.get(config_toml(), "jails", %{}) - - @doc """ - Path to the directory where all VM sockets are held. - - Must be stable across all nodes, and must be a directory. If it does not exist, `Hyper.Node` - will attempt to create one. - """ - @spec socket_dir :: Path.t() - def socket_dir, do: Path.join(work_dir(), "socks") - - @doc """ - Range in which `Hyper` allocates uid/gids: each VM gets a fresh uid/gid pair in - this range. Critical that no other process on the system uses this range. - - Read from `[jails] uid_gid_range` (a `[min, max]` array) in `#{@config_path}` — - the same file the helper validates against, so the node only ever hands out uids - the helper will accept. Defaults to `#{inspect(@default_uid_gid_range)}` when absent. - """ - @spec uid_gid_range :: {integer(), integer()} - def uid_gid_range do - case Map.get(jails(), "uid_gid_range") do - [min, max] -> {min, max} - _ -> @default_uid_gid_range - end - end - - @doc """ - Location of all image layers on all nodes. - - Hyper expects you to keep your layers in a flat directory, which may be backed by anything you - like: a plain filesystem, an NFS drive. This registry only ever is used to find paths to layers - but not anything more. - - Must be stable across all nodes, and must be a directory. If it does not exist, `Hyper.Node` - will attempt to create one. - - Derived as `/layers`, so it follows `work_dir` from `#{@config_path}`. - """ - @spec layer_dir :: Path.t() - def layer_dir, do: Path.join(work_dir(), "layers") - - @doc """ - Path to the skopeo binary (used by `Hyper.Img.OciLoader` to pull OCI images). - Read from `[tools] skopeo` in `#{@config_path}`, default `#{@default_skopeo}` (on `PATH`). - """ - @spec skopeo_path :: String.t() - def skopeo_path, do: tool_path("skopeo", @default_skopeo) - - @doc """ - Operator-configured path to the umoci binary, or `nil` (the default) to let - `Hyper.Img.OciLoader.Umoci` download and manage a pinned default. Read from - `[tools] umoci` in `#{@config_path}`. - """ - @spec umoci_path :: String.t() | nil - def umoci_path, do: optional_tool_path("umoci") - - @doc """ - Path to the mke2fs binary (used by `Hyper.Img.OciLoader` to build the ext4 rootfs). - Read from `[tools] mke2fs` in `#{@config_path}`, default `#{@default_mke2fs}` (on `PATH`). - """ - @spec mke2fs_path :: String.t() - def mke2fs_path, do: tool_path("mke2fs", @default_mke2fs) - - @doc """ - Path to the setuid-root device helper (`hyper-suidhelper`). The node runs - unprivileged and routes every `losetup`/`dmsetup`/`blockdev` operation through - it. - - Read from `[tools] suidhelper` in `#{@config_path}`, default `#{@default_suid_helper}` - (the install path used by `mix suidhelper.install`), so an operator who installs - it elsewhere can override it per node without recompiling. - """ - @spec suid_helper :: String.t() - def suid_helper, do: tool_path("suidhelper", @default_suid_helper) - - @doc """ - Directory for per-VM scratch (writable-layer COW) files. Must be node-local and - writable. If it does not exist, `Hyper.Node` will attempt to create one. - """ - @spec scratch_dir :: Path.t() - def scratch_dir, do: Path.join(work_dir(), "scratch") - - @doc """ - Per-architecture vmlinux (guest kernel) image paths, keyed by `Sys.Arch.t()`. - The operator places the kernels on the host and points these at them; - `Hyper.Node.Vmlinux` resolves and validates them per node. - """ - @spec vmlinux :: %{optional(Sys.Arch.t()) => Path.t()} - # Runtime read, not `compile_env`: an unset map would inline a literal `%{}`, - # which the type checker proves makes every `Map.fetch/2` on it return `:error`. - def vmlinux, do: Application.get_env(:hyper, :vmlinux, %{}) -end diff --git a/test/hyper/img/oci_loader_test.exs b/test/hyper/img/oci_loader_test.exs index 89277779..d959a4dd 100644 --- a/test/hyper/img/oci_loader_test.exs +++ b/test/hyper/img/oci_loader_test.exs @@ -2,7 +2,7 @@ defmodule Hyper.Img.OciLoaderTest do use ExUnit.Case, async: false use ExUnitProperties - alias Hyper.Config + alias Hyper.Cfg.Dirs alias Hyper.Img.Db.{Blob, Repo} alias Hyper.Img.OciLoader alias Unit.Information @@ -35,7 +35,7 @@ defmodule Hyper.Img.OciLoaderTest do assert {:ok, id} = OciLoader.load("docker.io/library/busybox:1.36") - path = Path.join(Config.layer_dir(), "layer_#{id}.img") + path = Path.join(Dirs.layer_dir(), "layer_#{id}.img") assert File.exists?(path) assert File.stat!(path).size > 0 diff --git a/test/hyper/node/fire_vmm/vm_linux/provider_test.exs b/test/hyper/node/fire_vmm/vm_linux/provider_test.exs index 4bd99330..7ddf1db6 100644 --- a/test/hyper/node/fire_vmm/vm_linux/provider_test.exs +++ b/test/hyper/node/fire_vmm/vm_linux/provider_test.exs @@ -35,12 +35,12 @@ defmodule Hyper.Node.FireVMM.VmLinux.ProviderTest do test "default_path/1 resolves under the configured install dir", %{builds: _} do assert {:ok, path} = Provider.default_path(:x86_64) - assert path == Path.join(Hyper.Config.vmlinux_install_dir(), "vmlinux-x86_64-6.1") + assert path == Path.join(Hyper.Cfg.Dirs.vmlinux_install_dir(), "vmlinux-x86_64-6.1") end test "path/1 resolves a known build under the install dir" do assert Provider.path("x86_64-6.1") == - {:ok, Path.join(Hyper.Config.vmlinux_install_dir(), "vmlinux-x86_64-6.1")} + {:ok, Path.join(Hyper.Cfg.Dirs.vmlinux_install_dir(), "vmlinux-x86_64-6.1")} end test "path/1 rejects an unknown build name" do From 513dfd3cc42ba9dcf5f03ec71239faa4814cc71d Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 20:55:13 +0000 Subject: [PATCH 09/47] refactor(cfg): move Node.Config.Budget to Hyper.Cfg.Budget Rename and relocate Hyper.Node.Config.Budget to Hyper.Cfg.Budget, update the runtime.exs app-env key, and fix all aliases/call-sites. --- config/runtime.exs | 2 +- lib/hyper/{node/budget/config.ex => cfg/budget.ex} | 2 +- lib/hyper/img/db/gc/config.ex | 2 +- lib/hyper/node.ex | 2 +- lib/hyper/node/budget/hard.ex | 4 ++-- lib/hyper/node/budget/node_state.ex | 2 +- lib/hyper/node/budget/soft.ex | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) rename lib/hyper/{node/budget/config.ex => cfg/budget.ex} (98%) diff --git a/config/runtime.exs b/config/runtime.exs index 45dbba68..b6358ecd 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -2,7 +2,7 @@ import Config # Per-node resource budget. Lives in runtime config because it builds `Unit.*` # values, which are only loadable once the app's modules are on the code path. -config :hyper, Hyper.Node.Config.Budget, +config :hyper, Hyper.Cfg.Budget, mem_max: Unit.Information.gib(4), disk_max: Unit.Information.gib(4), cpu_max_load: 0.8, diff --git a/lib/hyper/node/budget/config.ex b/lib/hyper/cfg/budget.ex similarity index 98% rename from lib/hyper/node/budget/config.ex rename to lib/hyper/cfg/budget.ex index 30e6218c..fdcacaae 100644 --- a/lib/hyper/node/budget/config.ex +++ b/lib/hyper/cfg/budget.ex @@ -1,4 +1,4 @@ -defmodule Hyper.Node.Config.Budget do +defmodule Hyper.Cfg.Budget do @moduledoc """ This node's resource budget configuration. diff --git a/lib/hyper/img/db/gc/config.ex b/lib/hyper/img/db/gc/config.ex index 83dea705..e4739a4b 100644 --- a/lib/hyper/img/db/gc/config.ex +++ b/lib/hyper/img/db/gc/config.ex @@ -3,7 +3,7 @@ defmodule Hyper.Img.Db.Gc.Config do Configuration for the layer garbage collector (`Hyper.Img.Db.Gc`). Every field has a default, so configuration is optional - set only what you want - to change. Durations are `Unit.Time` values, so (like `Hyper.Node.Config.Budget`) + to change. Durations are `Unit.Time` values, so (like `Hyper.Cfg.Budget`) overrides belong in `config/runtime.exs`: config :hyper, Hyper.Img.Db.Gc.Config, diff --git a/lib/hyper/node.ex b/lib/hyper/node.ex index 6591cb22..a5e0f63b 100644 --- a/lib/hyper/node.ex +++ b/lib/hyper/node.ex @@ -143,7 +143,7 @@ defmodule Hyper.Node do @spec test_system :: :ok | {:error, term()} def test_system do - with {:ok, _} <- Hyper.Node.Config.Budget.load(), + with {:ok, _} <- Hyper.Cfg.Budget.load(), :ok <- Hyper.Node.FireVMM.Provider.ensure_installed(), :ok <- Hyper.Node.FireVMM.VmLinux.Provider.ensure_installed(), :ok <- Hyper.Node.Vmlinux.test_system(), diff --git a/lib/hyper/node/budget/hard.ex b/lib/hyper/node/budget/hard.ex index 8f8e6916..fcd325cb 100644 --- a/lib/hyper/node/budget/hard.ex +++ b/lib/hyper/node/budget/hard.ex @@ -6,7 +6,7 @@ defmodule Hyper.Node.Budget.Hard do "Hard" means the limits are inviolable: a reservation that would push the running total past the node's configured `mem_max`/`disk_max` - (`Hyper.Node.Config.Budget`) is refused outright. Callers reserve through + (`Hyper.Cfg.Budget`) is refused outright. Callers reserve through `with_budget/2`, which holds the budget for the duration of the callback and releases it afterwards. """ @@ -15,7 +15,7 @@ defmodule Hyper.Node.Budget.Hard do use Unit.Operators use OpenTelemetryDecorator - alias Hyper.Node.Config.Budget, as: Config + alias Hyper.Cfg.Budget, as: Config alias Hyper.Vm.Instance defmodule State do diff --git a/lib/hyper/node/budget/node_state.ex b/lib/hyper/node/budget/node_state.ex index ba7921ea..f80b5e7d 100644 --- a/lib/hyper/node/budget/node_state.ex +++ b/lib/hyper/node/budget/node_state.ex @@ -11,7 +11,7 @@ defmodule Hyper.Node.Budget.NodeState do """ alias Hyper.Node.Budget.Hard - alias Hyper.Node.Config.Budget, as: Config + alias Hyper.Cfg.Budget, as: Config alias Hyper.Vm.Instance.Spec alias Sys.Mon alias Sys.Mon.Server.Reading diff --git a/lib/hyper/node/budget/soft.ex b/lib/hyper/node/budget/soft.ex index 758d9699..da50d2f0 100644 --- a/lib/hyper/node/budget/soft.ex +++ b/lib/hyper/node/budget/soft.ex @@ -7,7 +7,7 @@ defmodule Hyper.Node.Budget.Soft do The soft metrics are the ones whose overcommitment degrades speed rather than correctness: CPU utilization, disk bandwidth, and network bandwidth. For each, - the node carries a load ceiling in `Hyper.Node.Config.Budget` (e.g. "never + the node carries a load ceiling in `Hyper.Cfg.Budget` (e.g. "never schedule onto a machine already past 80% CPU"). A spec is admissible only if the measured load plus the spec's nominal demand stays under that ceiling on every metric. @@ -19,7 +19,7 @@ defmodule Hyper.Node.Budget.Soft do use Unit.Operators use OpenTelemetryDecorator - alias Hyper.Node.Config.Budget, as: Config + alias Hyper.Cfg.Budget, as: Config alias Hyper.Vm.Instance alias Sys.Mon alias Sys.Mon.Server.Reading From 5dca93db1179a097377f414b74d70f020f61d532 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 20:58:23 +0000 Subject: [PATCH 10/47] refactor(cfg): move Node.Config.Img to Hyper.Cfg.Img --- lib/hyper/{node/img/config.ex => cfg/img.ex} | 2 +- lib/hyper/node/img/thin_pool.ex | 2 +- lib/hyper/suid_helper/dmsetup.ex | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename lib/hyper/{node/img/config.ex => cfg/img.ex} (97%) diff --git a/lib/hyper/node/img/config.ex b/lib/hyper/cfg/img.ex similarity index 97% rename from lib/hyper/node/img/config.ex rename to lib/hyper/cfg/img.ex index e0fec815..7d554a8c 100644 --- a/lib/hyper/node/img/config.ex +++ b/lib/hyper/cfg/img.ex @@ -1,4 +1,4 @@ -defmodule Hyper.Node.Config.Img do +defmodule Hyper.Cfg.Img do @moduledoc """ This node's image storage configuration: the device-mapper geometry behind the read-only layer chain (dm-snapshot) and the per-VM writable layers (dm-thin). diff --git a/lib/hyper/node/img/thin_pool.ex b/lib/hyper/node/img/thin_pool.ex index bda11d7e..7a50f7ec 100644 --- a/lib/hyper/node/img/thin_pool.ex +++ b/lib/hyper/node/img/thin_pool.ex @@ -13,7 +13,7 @@ defmodule Hyper.Node.Img.ThinPool do use GenServer - alias Hyper.Node.Config.Img, as: ImgConfig + alias Hyper.Cfg.Img, as: ImgConfig alias Hyper.SuidHelper alias Unit.Information diff --git a/lib/hyper/suid_helper/dmsetup.ex b/lib/hyper/suid_helper/dmsetup.ex index 935ef84f..c8dabc88 100644 --- a/lib/hyper/suid_helper/dmsetup.ex +++ b/lib/hyper/suid_helper/dmsetup.ex @@ -18,7 +18,7 @@ defmodule Hyper.SuidHelper.Dmsetup do {:ok, Path.t()} | {:error, err()} @decorate with_span("Hyper.SuidHelper.Dmsetup.create_snapshot", include: [:name]) def create_snapshot(name, origin_dev, cow_dev, sectors) do - table = snapshot_table(origin_dev, cow_dev, sectors, Hyper.Node.Config.Img.chunk_sectors()) + table = snapshot_table(origin_dev, cow_dev, sectors, Hyper.Cfg.Img.chunk_sectors()) create(name, table, ["--readonly"]) end From 2ce426962d3972ae2d1eb71d782a2e212a5ae0e4 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 21:00:39 +0000 Subject: [PATCH 11/47] refactor(cfg): move Hyper.Grpc.Config to Hyper.Cfg.Grpc --- lib/hyper/cfg/grpc.ex | 57 ++++++++++++++++++++++++++++++++++++++++++ lib/hyper/grpc.ex | 58 +------------------------------------------ 2 files changed, 58 insertions(+), 57 deletions(-) create mode 100644 lib/hyper/cfg/grpc.ex diff --git a/lib/hyper/cfg/grpc.ex b/lib/hyper/cfg/grpc.ex new file mode 100644 index 00000000..6210512f --- /dev/null +++ b/lib/hyper/cfg/grpc.ex @@ -0,0 +1,57 @@ +defmodule Hyper.Cfg.Grpc do + @moduledoc """ + gRPC server configuration, read from application env into a struct: + + config :hyper, Hyper.Cfg.Grpc, + enabled: true, + port: 50_051, + cred: GRPC.Credential.new(ssl: [certfile: "/path/cert.pem", keyfile: "/path/key.pem"]) + + Fields: + + * `:enabled` -- whether the server starts. Defaults to `false`. + * `:port` -- the listen port. Defaults to `50051`. + * `:cred` -- a `GRPC.Credential` for TLS, or `nil` (the default) for + plaintext. + * `:adapter_opts` -- options forwarded to the server adapter, e.g. + `[ip: {0, 0, 0, 0}]`. + + Build the credential where you load your keys (e.g. `config/runtime.exs`); + Hyper never reads the filesystem on your behalf. + + > #### Co-located nodes {: .info} + > + > Every node binds `:port`. Running multiple nodes on one host (e.g. a local + > cluster) requires giving each a distinct port via its own config. + """ + + @default_port 50_051 + + defstruct enabled: false, port: @default_port, cred: nil, adapter_opts: [] + + @type t :: %__MODULE__{ + enabled: boolean(), + port: :inet.port_number(), + cred: GRPC.Credential.t() | nil, + adapter_opts: keyword() + } + + @doc "Load the gRPC server configuration from application env." + @spec load() :: t() + def load, do: struct!(__MODULE__, Application.get_env(:hyper, __MODULE__, [])) + + @doc """ + The `GRPC.Server.Supervisor` options for this config: the endpoint and port, + plus the TLS credential and adapter options when set. + """ + @spec server_options(t()) :: keyword() + def server_options(%__MODULE__{} = config) do + [endpoint: Hyper.Grpc.Endpoint, port: config.port, start_server: true] + |> put_unless(:cred, config.cred, nil) + |> put_unless(:adapter_opts, config.adapter_opts, []) + end + + @spec put_unless(keyword(), atom(), term(), term()) :: keyword() + defp put_unless(opts, _key, skip, skip), do: opts + defp put_unless(opts, key, value, _skip), do: Keyword.put(opts, key, value) +end diff --git a/lib/hyper/grpc.ex b/lib/hyper/grpc.ex index 65603551..450bd609 100644 --- a/lib/hyper/grpc.ex +++ b/lib/hyper/grpc.ex @@ -9,63 +9,7 @@ defmodule Hyper.Grpc do `Hyper.Grpc.V0.Hyper.Stub` together with `connect/2`. """ - defmodule Config do - @moduledoc """ - gRPC server configuration, read from application env into a struct: - - config :hyper, Hyper.Grpc.Config, - enabled: true, - port: 50_051, - cred: GRPC.Credential.new(ssl: [certfile: "/path/cert.pem", keyfile: "/path/key.pem"]) - - Fields: - - * `:enabled` -- whether the server starts. Defaults to `false`. - * `:port` -- the listen port. Defaults to `50051`. - * `:cred` -- a `GRPC.Credential` for TLS, or `nil` (the default) for - plaintext. - * `:adapter_opts` -- options forwarded to the server adapter, e.g. - `[ip: {0, 0, 0, 0}]`. - - Build the credential where you load your keys (e.g. `config/runtime.exs`); - Hyper never reads the filesystem on your behalf. - - > #### Co-located nodes {: .info} - > - > Every node binds `:port`. Running multiple nodes on one host (e.g. a local - > cluster) requires giving each a distinct port via its own config. - """ - - @default_port 50_051 - - defstruct enabled: false, port: @default_port, cred: nil, adapter_opts: [] - - @type t :: %__MODULE__{ - enabled: boolean(), - port: :inet.port_number(), - cred: GRPC.Credential.t() | nil, - adapter_opts: keyword() - } - - @doc "Load the gRPC server configuration from application env." - @spec load() :: t() - def load, do: struct!(__MODULE__, Application.get_env(:hyper, __MODULE__, [])) - - @doc """ - The `GRPC.Server.Supervisor` options for this config: the endpoint and port, - plus the TLS credential and adapter options when set. - """ - @spec server_options(t()) :: keyword() - def server_options(%__MODULE__{} = config) do - [endpoint: Hyper.Grpc.Endpoint, port: config.port, start_server: true] - |> put_unless(:cred, config.cred, nil) - |> put_unless(:adapter_opts, config.adapter_opts, []) - end - - @spec put_unless(keyword(), atom(), term(), term()) :: keyword() - defp put_unless(opts, _key, skip, skip), do: opts - defp put_unless(opts, key, value, _skip), do: Keyword.put(opts, key, value) - end + alias Hyper.Cfg.Grpc, as: Config @doc """ The gRPC server's supervisor child, or `[]` when the server is disabled (the From 22e77cf49b01b81800f47d1a8e791813928c889d Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 21:03:39 +0000 Subject: [PATCH 12/47] refactor(cfg): move Img.Db.Gc.Config to Hyper.Cfg.Gc --- lib/hyper/{img/db/gc/config.ex => cfg/gc.ex} | 4 ++-- lib/hyper/img/db/gc.ex | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) rename lib/hyper/{img/db/gc/config.ex => cfg/gc.ex} (96%) diff --git a/lib/hyper/img/db/gc/config.ex b/lib/hyper/cfg/gc.ex similarity index 96% rename from lib/hyper/img/db/gc/config.ex rename to lib/hyper/cfg/gc.ex index e4739a4b..86f572f6 100644 --- a/lib/hyper/img/db/gc/config.ex +++ b/lib/hyper/cfg/gc.ex @@ -1,4 +1,4 @@ -defmodule Hyper.Img.Db.Gc.Config do +defmodule Hyper.Cfg.Gc do @moduledoc """ Configuration for the layer garbage collector (`Hyper.Img.Db.Gc`). @@ -6,7 +6,7 @@ defmodule Hyper.Img.Db.Gc.Config do to change. Durations are `Unit.Time` values, so (like `Hyper.Cfg.Budget`) overrides belong in `config/runtime.exs`: - config :hyper, Hyper.Img.Db.Gc.Config, + config :hyper, Hyper.Cfg.Gc, enabled: true, sweep_interval: Unit.Time.s(30), grace_period: Unit.Time.s(60 * 60) diff --git a/lib/hyper/img/db/gc.ex b/lib/hyper/img/db/gc.ex index 02434c27..c16abef8 100644 --- a/lib/hyper/img/db/gc.ex +++ b/lib/hyper/img/db/gc.ex @@ -33,7 +33,8 @@ defmodule Hyper.Img.Db.Gc do alias Hyper.Cluster.Routing alias Hyper.Img.Db.{Blob, ImageLayer, Repo} - alias Hyper.Img.Db.Gc.{Config, Sweep} + alias Hyper.Cfg.Gc, as: Config + alias Hyper.Img.Db.Gc.Sweep alias Hyper.Node.Layer.Repo, as: LayerRepo @singleton_key {:singleton, :layer_gc} From bd85067289e644bc6868a788807c3cae64a34fde Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 21:10:17 +0000 Subject: [PATCH 13/47] feat(cfg): Hyper.Cfg.Mon centralizes monitor sampling cadence Delete the hardcoded @period_ms/@tau_s module attributes from the four /proc monitors and replace period/0 + tau/0 with delegations to the new Hyper.Cfg.Mon module. Operators can now override sampling cadence per metric via config :hyper, Hyper.Cfg.Mon, cpu: [period_ms: .., tau_s: ..]. --- lib/hyper/cfg/mon.ex | 41 +++++++++++++++++++++++++++++++++++++ lib/sys/mon/cpu.ex | 14 ++++++------- lib/sys/mon/disk_bw.ex | 14 ++++++------- lib/sys/mon/mem.ex | 14 ++++++------- lib/sys/mon/net_bw.ex | 14 ++++++------- test/hyper/cfg/mon_test.exs | 19 +++++++++++++++++ 6 files changed, 84 insertions(+), 32 deletions(-) create mode 100644 lib/hyper/cfg/mon.ex create mode 100644 test/hyper/cfg/mon_test.exs diff --git a/lib/hyper/cfg/mon.ex b/lib/hyper/cfg/mon.ex new file mode 100644 index 00000000..f4d53867 --- /dev/null +++ b/lib/hyper/cfg/mon.ex @@ -0,0 +1,41 @@ +defmodule Hyper.Cfg.Mon do + @moduledoc """ + Sampling cadence for the `/proc`-backed monitors (`Sys.Mon.*`). + + Each metric has a deliberately co-prime sampling period (so the four monitors + rarely sample on the same tick) and an EWMA time constant. Operators may + override per metric via `config :hyper, Hyper.Cfg.Mon, cpu: [period_ms: .., + tau_s: ..]` (plain integers, ms and s); the defaults below are the tuned + values. The accessors return `Unit.Time` quantities — `Sys.Mon.Server` + consumes them via `Unit.Time.as_ms/1`. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @type metric :: :cpu | :mem | :disk_bw | :net_bw + + @defaults %{ + cpu: [period_ms: 23, tau_s: 30], + mem: [period_ms: 29, tau_s: 30], + disk_bw: [period_ms: 31, tau_s: 20], + net_bw: [period_ms: 37, tau_s: 20] + } + + @doc "Sampling period for `metric`." + @spec period(metric()) :: Unit.Time.t() + def period(metric), do: Unit.Time.ms(field(metric, :period_ms)) + + @doc "EWMA smoothing time constant for `metric`." + @spec tau(metric()) :: Unit.Time.t() + def tau(metric), do: Unit.Time.s(field(metric, :tau_s)) + + @spec field(metric(), atom()) :: pos_integer() + defp field(metric, key) do + default = @defaults |> Map.fetch!(metric) |> Keyword.fetch!(key) + + case get_cfg(runtime: {__MODULE__, metric}, default: []) do + kw when is_list(kw) -> Keyword.get(kw, key, default) + _ -> default + end + end +end diff --git a/lib/sys/mon/cpu.ex b/lib/sys/mon/cpu.ex index 47d6d312..5eac7eb6 100644 --- a/lib/sys/mon/cpu.ex +++ b/lib/sys/mon/cpu.ex @@ -3,26 +3,24 @@ defmodule Sys.Mon.Cpu do alias Sys.Linux.Proc.Stat alias Sys.Mon.Server - alias Unit.Time - - @period_ms 23 - @tau_s 30 @moduledoc """ Monitors instantaneous CPU utilization (the soft beta_vcpus signal). - Samples `/proc/stat` every #{@period_ms} ms and reports the busy fraction + Samples `/proc/stat` every 23 ms and reports the busy fraction (`0.0..1.0`, normalized across all cores) between consecutive reads - never the load average, which has different semantics. The first read only establishes a - baseline (`:skip`). Readings are smoothed with a #{@tau_s}-second time constant + baseline (`:skip`). Readings are smoothed with a 30-second time constant (sampling fast only de-noises the filter; the smoothing window is set by `tau`). """ @impl true - def period, do: Time.ms(@period_ms) + @spec period :: Unit.Time.t() + def period, do: Hyper.Cfg.Mon.period(:cpu) @impl true - def tau, do: Time.s(@tau_s) + @spec tau :: Unit.Time.t() + def tau, do: Hyper.Cfg.Mon.tau(:cpu) @doc "The latest instantaneous + filtered CPU utilization (fractions `0.0..1.0`)." @spec value() :: Server.Reading.t() diff --git a/lib/sys/mon/disk_bw.ex b/lib/sys/mon/disk_bw.ex index 5cc97c73..899696af 100644 --- a/lib/sys/mon/disk_bw.ex +++ b/lib/sys/mon/disk_bw.ex @@ -5,25 +5,23 @@ defmodule Sys.Mon.DiskBw do alias Sys.Linux.Proc.Diskstats alias Sys.Mon.Server alias Unit.Bandwidth - alias Unit.Time - - @period_ms 31 - @tau_s 20 @moduledoc """ Monitors instantaneous disk bandwidth (the soft beta_disk_bw signal). Samples cumulative read+write bytes across whole physical disks from - `/proc/diskstats` every #{@period_ms} ms and differentiates them into bytes/sec + `/proc/diskstats` every 31 ms and differentiates them into bytes/sec via `Controls.Rate` (the first read only establishes a baseline). The rate series - is smoothed with a #{@tau_s}-second time constant. Readings are `Unit.Bandwidth`. + is smoothed with a 20-second time constant. Readings are `Unit.Bandwidth`. """ @impl true - def period, do: Time.ms(@period_ms) + @spec period :: Unit.Time.t() + def period, do: Hyper.Cfg.Mon.period(:disk_bw) @impl true - def tau, do: Time.s(@tau_s) + @spec tau :: Unit.Time.t() + def tau, do: Hyper.Cfg.Mon.tau(:disk_bw) @doc "The latest instantaneous + filtered disk bandwidth (`Unit.Bandwidth` readings)." @spec value() :: Server.Reading.t() diff --git a/lib/sys/mon/mem.ex b/lib/sys/mon/mem.ex index 603e5133..86d58201 100644 --- a/lib/sys/mon/mem.ex +++ b/lib/sys/mon/mem.ex @@ -4,25 +4,23 @@ defmodule Sys.Mon.Mem do alias Sys.Linux.Proc.Meminfo alias Sys.Mon.Server alias Unit.Information - alias Unit.Time - - @period_ms 29 - @tau_s 30 @moduledoc """ Monitors instantaneous memory pressure. - Samples `/proc/meminfo` every #{@period_ms} ms and reports *used* memory as - `MemTotal - MemAvailable`, smoothed with a #{@tau_s}-second time constant. Although + Samples `/proc/meminfo` every 29 ms and reports *used* memory as + `MemTotal - MemAvailable`, smoothed with a 30-second time constant. Although memory is an alpha (hard) budget tracked from VM specs, the live figure is useful for detecting actual pressure. Readings are `Unit.Information`. """ @impl true - def period, do: Time.ms(@period_ms) + @spec period :: Unit.Time.t() + def period, do: Hyper.Cfg.Mon.period(:mem) @impl true - def tau, do: Time.s(@tau_s) + @spec tau :: Unit.Time.t() + def tau, do: Hyper.Cfg.Mon.tau(:mem) @doc "The latest instantaneous + filtered used memory (`Unit.Information` readings)." @spec value() :: Server.Reading.t() diff --git a/lib/sys/mon/net_bw.ex b/lib/sys/mon/net_bw.ex index c877a52f..576a39c0 100644 --- a/lib/sys/mon/net_bw.ex +++ b/lib/sys/mon/net_bw.ex @@ -5,25 +5,23 @@ defmodule Sys.Mon.NetBw do alias Sys.Linux.Proc.NetDev alias Sys.Mon.Server alias Unit.Bandwidth - alias Unit.Time - - @period_ms 37 - @tau_s 20 @moduledoc """ Monitors instantaneous network bandwidth (the soft beta_net_bw signal). Samples cumulative rx+tx bytes across physical interfaces from `/proc/net/dev` - every #{@period_ms} ms and differentiates them into bytes/sec via `Controls.Rate` + every 37 ms and differentiates them into bytes/sec via `Controls.Rate` (the first read only establishes a baseline). The rate series is smoothed with a - #{@tau_s}-second time constant. Readings are `Unit.Bandwidth`. + 20-second time constant. Readings are `Unit.Bandwidth`. """ @impl true - def period, do: Time.ms(@period_ms) + @spec period :: Unit.Time.t() + def period, do: Hyper.Cfg.Mon.period(:net_bw) @impl true - def tau, do: Time.s(@tau_s) + @spec tau :: Unit.Time.t() + def tau, do: Hyper.Cfg.Mon.tau(:net_bw) @doc "The latest instantaneous + filtered network bandwidth (`Unit.Bandwidth` readings)." @spec value() :: Server.Reading.t() diff --git a/test/hyper/cfg/mon_test.exs b/test/hyper/cfg/mon_test.exs new file mode 100644 index 00000000..8f0a245f --- /dev/null +++ b/test/hyper/cfg/mon_test.exs @@ -0,0 +1,19 @@ +defmodule Hyper.Cfg.MonTest do + use ExUnit.Case, async: true + + alias Hyper.Cfg.Mon + + test "default sampling periods stay co-prime per metric" do + assert Mon.period(:cpu) == Unit.Time.ms(23) + assert Mon.period(:mem) == Unit.Time.ms(29) + assert Mon.period(:disk_bw) == Unit.Time.ms(31) + assert Mon.period(:net_bw) == Unit.Time.ms(37) + end + + test "default EWMA time constants" do + assert Mon.tau(:cpu) == Unit.Time.s(30) + assert Mon.tau(:mem) == Unit.Time.s(30) + assert Mon.tau(:disk_bw) == Unit.Time.s(20) + assert Mon.tau(:net_bw) == Unit.Time.s(20) + end +end From 5717c4b16eff3f9797eb650320de69f92dfcde37 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 21:18:42 +0000 Subject: [PATCH 14/47] feat(cfg): Hyper.Cfg.Timeouts centralizes idle + call timeouts --- lib/hyper/cfg/timeouts.ex | 25 +++++++++++++++++++++++++ lib/hyper/node/fire_vmm/client.ex | 4 +--- lib/hyper/node/img/mutable.ex | 4 +--- lib/hyper/node/img/server.ex | 5 +---- lib/hyper/node/layer/server.ex | 6 +----- test/hyper/cfg/timeouts_test.exs | 15 +++++++++++++++ 6 files changed, 44 insertions(+), 15 deletions(-) create mode 100644 lib/hyper/cfg/timeouts.ex create mode 100644 test/hyper/cfg/timeouts_test.exs diff --git a/lib/hyper/cfg/timeouts.ex b/lib/hyper/cfg/timeouts.ex new file mode 100644 index 00000000..11aeea9a --- /dev/null +++ b/lib/hyper/cfg/timeouts.ex @@ -0,0 +1,25 @@ +defmodule Hyper.Cfg.Timeouts do + @moduledoc """ + Teardown and RPC timeouts. The idle grace is how long a read-only image + (`:img`), a layer mount (`:layer`), or a mutable layer (`:mutable`) lingers + with no users before it is torn down; `fire_call_ms/0` caps a single + Firecracker API call. Override via `config :hyper, Hyper.Cfg.Timeouts, ...`. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @type scope :: :img | :layer | :mutable + + @doc "Idle grace before teardown for `scope`, in milliseconds (default 30s)." + @spec idle_ms(scope()) :: pos_integer() + def idle_ms(scope) when scope in [:img, :layer, :mutable] do + case get_cfg(runtime: {__MODULE__, :idle_ms}, default: []) do + kw when is_list(kw) -> Keyword.get(kw, scope, :timer.seconds(30)) + _ -> :timer.seconds(30) + end + end + + @doc "Per-call Firecracker API timeout, in milliseconds (default 35s)." + @spec fire_call_ms :: pos_integer() + def fire_call_ms, do: get_cfg(runtime: {__MODULE__, :fire_call_ms}, default: 35_000) +end diff --git a/lib/hyper/node/fire_vmm/client.ex b/lib/hyper/node/fire_vmm/client.ex index a336b095..1314ea56 100644 --- a/lib/hyper/node/fire_vmm/client.ex +++ b/lib/hyper/node/fire_vmm/client.ex @@ -33,8 +33,6 @@ defmodule Hyper.Node.FireVMM.Client do alias Hyper.Node.FireVMM.Jailer - @call_timeout 35_000 - defmodule Opts do @moduledoc """ Start options for `Hyper.Node.FireVMM.Client`. Only `:vm_id` is required; @@ -74,7 +72,7 @@ defmodule Hyper.Node.FireVMM.Client do @doc "Run a generated operation against this VM's daemon, serialized." @spec run(GenServer.server(), (keyword() -> result)) :: result when result: var def run(server, op_fun) when is_function(op_fun, 1) do - GenServer.call(server, {:run, op_fun}, @call_timeout) + GenServer.call(server, {:run, op_fun}, Hyper.Cfg.Timeouts.fire_call_ms()) end @impl true diff --git a/lib/hyper/node/img/mutable.ex b/lib/hyper/node/img/mutable.ex index 386e36f0..a516ff58 100644 --- a/lib/hyper/node/img/mutable.ex +++ b/lib/hyper/node/img/mutable.ex @@ -24,8 +24,6 @@ defmodule Hyper.Node.Img.Mutable do use OpenTelemetryDecorator - @idle_timeout_ms :timer.seconds(30) - defmodule Opts do @moduledoc false @enforce_keys [:img_id, :vm_id] @@ -151,7 +149,7 @@ defmodule Hyper.Node.Img.Mutable do defp arm_idle(state) do state = cancel_idle(state) - %{state | idle_ref: Process.send_after(self(), :idle_timeout, @idle_timeout_ms)} + %{state | idle_ref: Process.send_after(self(), :idle_timeout, Hyper.Cfg.Timeouts.idle_ms(:mutable))} end defp cancel_idle(%State{idle_ref: nil} = state), do: state diff --git a/lib/hyper/node/img/server.ex b/lib/hyper/node/img/server.ex index d51fe3ba..57364bb5 100644 --- a/lib/hyper/node/img/server.ex +++ b/lib/hyper/node/img/server.ex @@ -21,9 +21,6 @@ defmodule Hyper.Node.Img.Server do use OpenTelemetryDecorator - # Grace period after the last holder leaves before the image is torn down. - @idle_timeout_ms :timer.seconds(30) - defmodule State do @moduledoc false @@ -228,7 +225,7 @@ defmodule Hyper.Node.Img.Server do @spec arm_idle(State.t()) :: State.t() defp arm_idle(state) do state = cancel_idle(state) - %{state | idle_ref: Process.send_after(self(), :idle_timeout, @idle_timeout_ms)} + %{state | idle_ref: Process.send_after(self(), :idle_timeout, Hyper.Cfg.Timeouts.idle_ms(:img))} end @spec cancel_idle(State.t()) :: State.t() diff --git a/lib/hyper/node/layer/server.ex b/lib/hyper/node/layer/server.ex index 188e135b..e80848fd 100644 --- a/lib/hyper/node/layer/server.ex +++ b/lib/hyper/node/layer/server.ex @@ -15,10 +15,6 @@ defmodule Hyper.Node.Layer.Server do alias Hyper.Node.Layer.Repo alias Hyper.SuidHelper - # Grace period after the last holder leaves before the layer is unmounted. Keeps - # bursty acquire/release cycles from thrashing the mount. - @idle_timeout_ms :timer.seconds(30) - defmodule State do @moduledoc false @@ -152,7 +148,7 @@ defmodule Hyper.Node.Layer.Server do @spec arm_idle(State.t()) :: State.t() defp arm_idle(state) do state = cancel_idle(state) - %{state | idle_ref: Process.send_after(self(), :idle_timeout, @idle_timeout_ms)} + %{state | idle_ref: Process.send_after(self(), :idle_timeout, Hyper.Cfg.Timeouts.idle_ms(:layer))} end @spec cancel_idle(State.t()) :: State.t() diff --git a/test/hyper/cfg/timeouts_test.exs b/test/hyper/cfg/timeouts_test.exs new file mode 100644 index 00000000..e7f2a895 --- /dev/null +++ b/test/hyper/cfg/timeouts_test.exs @@ -0,0 +1,15 @@ +defmodule Hyper.Cfg.TimeoutsTest do + use ExUnit.Case, async: true + + alias Hyper.Cfg.Timeouts + + test "idle grace defaults to 30s for every teardown scope" do + assert Timeouts.idle_ms(:img) == :timer.seconds(30) + assert Timeouts.idle_ms(:layer) == :timer.seconds(30) + assert Timeouts.idle_ms(:mutable) == :timer.seconds(30) + end + + test "firecracker API call timeout default" do + assert Timeouts.fire_call_ms() == 35_000 + end +end From f326e586a4f22acd706f27d50c5e908c0c621cea Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 21:23:27 +0000 Subject: [PATCH 15/47] feat(cfg): read-only facades for telemetry/db/cluster config --- lib/hyper/application.ex | 2 +- lib/hyper/cfg/cluster.ex | 6 ++++++ lib/hyper/cfg/db.ex | 6 ++++++ lib/hyper/cfg/telemetry.ex | 9 ++++++++ test/hyper/cfg/facades_test.exs | 37 +++++++++++++++++++++++++++++++++ 5 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 lib/hyper/cfg/cluster.ex create mode 100644 lib/hyper/cfg/db.ex create mode 100644 lib/hyper/cfg/telemetry.ex create mode 100644 test/hyper/cfg/facades_test.exs diff --git a/lib/hyper/application.ex b/lib/hyper/application.ex index 96d28b3e..4a0a280b 100644 --- a/lib/hyper/application.ex +++ b/lib/hyper/application.ex @@ -12,7 +12,7 @@ defmodule Hyper.Application do # the repo's default telemetry_prefix (its module path, underscored). _ = OpentelemetryEcto.setup([:hyper, :img, :db, :repo]) - topologies = Application.get_env(:libcluster, :topologies, []) + topologies = Hyper.Cfg.Cluster.topologies() children = [ diff --git a/lib/hyper/cfg/cluster.ex b/lib/hyper/cfg/cluster.ex new file mode 100644 index 00000000..4d13dc4c --- /dev/null +++ b/lib/hyper/cfg/cluster.ex @@ -0,0 +1,6 @@ +defmodule Hyper.Cfg.Cluster do + @moduledoc "Read-only view of the libcluster topology (`config :libcluster`)." + + @spec topologies :: keyword() + def topologies, do: Application.get_env(:libcluster, :topologies, []) +end diff --git a/lib/hyper/cfg/db.ex b/lib/hyper/cfg/db.ex new file mode 100644 index 00000000..597de8f2 --- /dev/null +++ b/lib/hyper/cfg/db.ex @@ -0,0 +1,6 @@ +defmodule Hyper.Cfg.Db do + @moduledoc "Read-only view of the image-DB Ecto repo config." + + @spec repo_opts :: keyword() + def repo_opts, do: Application.get_env(:hyper, Hyper.Img.Db.Repo, []) +end diff --git a/lib/hyper/cfg/telemetry.ex b/lib/hyper/cfg/telemetry.ex new file mode 100644 index 00000000..4c2f9a3a --- /dev/null +++ b/lib/hyper/cfg/telemetry.ex @@ -0,0 +1,9 @@ +defmodule Hyper.Cfg.Telemetry do + @moduledoc "Read-only view of the OpenTelemetry exporter config." + + @spec exporter :: atom() + def exporter, do: Application.get_env(:opentelemetry, :traces_exporter, :none) + + @spec otlp_endpoint :: String.t() | nil + def otlp_endpoint, do: Application.get_env(:opentelemetry_exporter, :otlp_endpoint) +end diff --git a/test/hyper/cfg/facades_test.exs b/test/hyper/cfg/facades_test.exs new file mode 100644 index 00000000..8d4960e7 --- /dev/null +++ b/test/hyper/cfg/facades_test.exs @@ -0,0 +1,37 @@ +defmodule Hyper.Cfg.FacadesTest do + use ExUnit.Case, async: false + + test "Cluster.topologies reads :libcluster app env" do + assert is_list(Hyper.Cfg.Cluster.topologies()) + end + + test "Db.repo_opts reads the Ecto repo config" do + assert Keyword.keyword?(Hyper.Cfg.Db.repo_opts()) + end + + test "Telemetry.exporter reflects the configured traces_exporter" do + prior = Application.get_env(:opentelemetry, :traces_exporter) + + on_exit(fn -> + if prior == nil, + do: Application.delete_env(:opentelemetry, :traces_exporter), + else: Application.put_env(:opentelemetry, :traces_exporter, prior) + end) + + Application.put_env(:opentelemetry, :traces_exporter, :none) + assert Hyper.Cfg.Telemetry.exporter() == :none + end + + test "Telemetry.otlp_endpoint reads the configured endpoint" do + prior = Application.get_env(:opentelemetry_exporter, :otlp_endpoint) + + on_exit(fn -> + if prior == nil, + do: Application.delete_env(:opentelemetry_exporter, :otlp_endpoint), + else: Application.put_env(:opentelemetry_exporter, :otlp_endpoint, prior) + end) + + Application.put_env(:opentelemetry_exporter, :otlp_endpoint, "http://x:4318") + assert Hyper.Cfg.Telemetry.otlp_endpoint() == "http://x:4318" + end +end From 66fb7308b590846d68077e6177c47053b7bf42e7 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 21:33:29 +0000 Subject: [PATCH 16/47] =?UTF-8?q?refactor(cfg):=20polish=20=E2=80=94=20ali?= =?UTF-8?q?as=20ordering,=20format,=20batched=20review=20nits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - alphabetize aliases in node_state.ex/gc.ex/tools_test.exs (credo --strict) - mix format the longer Hyper.Cfg.* call-sites (oci_loader.ex, img/server.ex) - restore anti-thrash rationale comment in layer/server.ex moduledoc - add Jails.range_from/1 non-integer-list refusal test - drop unused Hyper.Cfg.Toml.path/0 (YAGNI) --- docs/cookbook/config.md | 403 +++++++++++----------------- lib/hyper/cfg.ex | 1 + lib/hyper/cfg/toml.ex | 4 - lib/hyper/img/db/gc.ex | 2 +- lib/hyper/img/oci_loader.ex | 4 +- lib/hyper/node/budget/node_state.ex | 2 +- lib/hyper/node/img/mutable.ex | 6 +- lib/hyper/node/img/server.ex | 6 +- lib/hyper/node/layer/server.ex | 9 +- test/hyper/cfg/jails_test.exs | 2 + test/hyper/cfg/tools_test.exs | 2 +- 11 files changed, 180 insertions(+), 261 deletions(-) diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md index cf59b7cf..31850efa 100644 --- a/docs/cookbook/config.md +++ b/docs/cookbook/config.md @@ -2,277 +2,182 @@ Configuring `Hyper` is done through four layers, in priority: - 1. Runtime `/etc/hyper/config.exs` is the canonical Elixir way to configure - the system. This allows you to inject arbitrary code to configure `Hyper`. - 2. `Hyper` will fall back to reading `/etc/hyper/config.toml` at runtime, on - bootup, on each node. - 3. `Hyper` will use its compile-time configuration through `config.ex`. - 4. `Hyper` will use defaults. +## Configuration Files -**Note that not all layers allow all configuration fields to be tweaked.** This -is usually done for security. +| File | Description | +|------|-------------| +| `/etc/hyper/config.exs` | The `config.exs` file is exlusively used by the unprivileged `hyper` application. The purpose of this file is to allow you to load configuration values at runtime. If you are using a secrets manager, this is the right place to load the secrets. Must be owned by `root` and only writeable by `root`. | +| `/etc/hyper/config.toml` | The `/etc/hyper/config.toml` file is used for static configuration. Unlike `config.exs`, it is used by both `Hyper` and `hyper-suidhelper` which means that it can impact the behavior of a process running under `root`. Must be owned by `root` and only writable by `root`. | +| Compile-Time `config.ex` | The compile-time configuration is generally used to fine-tune the performance of Hyper. You likely do not need to edit most of the configuration fields exposed by this file for day-to-day usage, but they are available for you to tweak. | +| Defaults | `Hyper` has a set of sane defaults for some, but not all config fields. | -## Configuration Files +**Note that not all layers allow all configuration fields to be tweaked.** Read +further for more details on where and how each configuration field is set. -### `/etc/hyper/config.exs` +## Configuration Fields -The `config.exs` file is exlusively used by the unprivileged `hyper` -application. The purpose of this file is to allow you to load configuration -values at runtime. If you are using a secrets manager, this is the right place -to load the secrets. +This section briefly outlines the configuration fields available in `Hyper`. +Note the keys are abbreviated for better layout: -### `/etc/hyper/config.toml` + - `config.exs` refers to `/etc/hyper/config.exs`. + - `config.toml` refers to `/etc/hyper/config.toml`. + - All keys under `config.exs` are written in short-hand form. The parent + group is given as the section title. For example, `.mke2fs` in the tool + configuration section expands to + `:hyper, Hyper.Config.Tools, mke2fs: "/path/to/mke2fs"`. -The `/etc/hyper/config.toml` file is used for static configuration. Unlike -`config.exs`, it is used by both `Hyper` and `hyper-suidhelper` which means -that it can impact the behavior of a process running under `root`. +### Root Keys (`Hyper.Config`, `-`) -### Compile-Time Config +| Config Key | `config.exs` | `config.toml` | Default | Notes | +|---------------|-------------------------|--------------------------|-----------------------------------|-------------------------------------------------------------------------| +| `work_dir` | - | `work_dir` | - | [Absolute Path](#absolute-path) where `Hyper` creates its working tree. | -The compile-time configuration is generally used to fine-tune the performance -of Hyper. You likely do not need to edit most of the configuration fields -exposed by this file for day-to-day usage, but they are available for you to -tweak. +### Tool Configuration (`Hyper.Config.Tools`, `[tools]`) -## Configuration Fields +Hyper relies on a large number of external tools, of which the paths are +configurable: -### Tool Configuration +| Config Key | `config.exs` | `config.toml` | Default | Notes | +|---------------|-------------------------|--------------------------|-------------------------------------|---------------------------------| +| `firecracker` | - | `.firecracker` | - | [Safe Path](#safe-path) | +| `jailer` | - | `.jailer` | - | [Safe Path](#safe-path) | +| `dmsetup` | - | `.dmsetup` | `"/usr/sbin/dmsetup"` | [Safe Path](#safe-path) | +| `losetup` | - | `.losetup` | `"/usr/sbin/losetup"` | [Safe Path](#safe-path) | +| `blockdev` | - | `.blockdev` | `"/usr/sbin/blockdev"` | [Safe Path](#safe-path) | +| `mke2fs` | `.mke2fs` | `.losetup` | `$PATH["mke2fs"]` | | +| `skopeo` | `.skopeo` | `.skopeo` | `$PATH["skopeo"]` | | +| `umoci` | `.umoci` | `.umoci` | Automatically downloaded. | | +| `suidhelper` | `.suidhelper` | `.suidhelper` | `"/usr/local/bin/hyper-suidhelper"` | [Absolute Path](#absolute-path) | -Hyper relies on a large number of external tools, all configured under the -`[tools]` table in `/etc/hyper/config.toml`. +### Jail Confinement (`-`, `[jails]`) -#### Privileged tools (run by the setuid helper) +| Config Key | `config.exs` | `config.toml` | Default | Notes | +|----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `cgroup` | - | `.cgroup` | `"hyper"` | Parent cgroup under which each VM's cgroup is nested. Each VM receives its own ephemeral cgroup which lives under the umbrella of this cgroup. | +| `uid_gid_range`| - | `.uid_gid_range` | - | [Range](#range) limiting the UID/GID values given to VMs. Each VM receives its own UID/GID pair, within these bounds. Must not be an existing user/group. | -| Tool | Required | Default | `/etc/hyper/config.toml` | -|------|----------|---------|--------------------------| -| `firecracker` | Yes | - | `tools.firecracker` | -| `jailer` | Yes | - | `tools.jailer` | -| `dmsetup` | No | `/usr/sbin/dmsetup` | `tools.dmsetup` | -| `losetup` | No | `/usr/sbin/losetup` | `tools.losetup` | -| `blockdev` | No | `/usr/sbin/blockdev` | `tools.blockdev` | +### gRPC Configuration (`Hyper.Config.Grpc`, `[grpc]`) -> #### Requirements {: .info} -> -> - These paths **can only** be configured through `/etc/hyper/config.toml`. -> Both `Hyper` and `hyper-setuidhelper` rely on these paths being identical. +Hyper supports a [gRPC](https://grpc.io/) interface enabling you to interface +with `Hyper` from any language. + +| Config Key | `config.exs` | `config.toml`| Default | Notes | +|------------|-------------------------------------|-------------------------|---------|--------------------------------------------------------------------------------------------------------| +| `enabled` | `.enabled` | `.enabled` | `false` | | +| `port` | `.port` | `.port` | `50051` | The port on which to serve the interface. | +| `cred` | `.cred` | `.cred` | `nil` | Either a `GRPC.Credential` or a TOML struct `{ cert = "/path/to/cert.pem", key = "/path/to/key.pem"}`. Cleartext mode when `nil`. | + +> #### Uniqueness {: .info} > -> - The paths **must** be given as absolute paths. -> - The basename **must** match the configuration, eg. `firecracker` must have -> a path `/foo/bar/firecracker`. -> - The tools must be owned by the `root` user. -> - The tools must be exlusively writable by `root`. - -#### Node tools (run by the unprivileged node) - -| Tool | Required | Default | `/etc/hyper/config.toml` | -|------|----------|---------|--------------------------| -| `skopeo` | No | `skopeo` (on `PATH`) | `tools.skopeo` | -| `mke2fs` | No | `mke2fs` (on `PATH`) | `tools.mke2fs` | -| `umoci` | No | downloaded + cached under `/redist/umoci` | `tools.umoci` | -| `suidhelper` | No | `/usr/local/bin/hyper-suidhelper` | `tools.suidhelper` | - -> #### These are not privileged {: .info} +> Note that if you choose to use a homogenous configuration across all your +> nodes and you enable the gRPC server on all of them, you will spawn multiple +> gRPC servers, one-per-node, in your cluster. > -> The node runs these directly as the unprivileged hyper user, so — unlike the -> privileged tools above — they carry **no** root-ownership or basename -> requirement. `skopeo`/`mke2fs` default to the bare name resolved on `PATH`; -> leave `umoci` unset to let Hyper download and cache a pinned release. They -> share the one `[tools]` table with the privileged binaries — the helper simply -> ignores the keys it does not own. +> This is perfectly legal, if you so desire, but it is important to note that +> you can also conditionally enable the `gRPC` server based on logic in your +> `config.exs`, for example, to only spawn it on your "main" server. -## The shared file: `/etc/hyper/config.toml` +### Telemetry Configuration (`Hyper.Config.Otel`, `[otel]`) -> #### Security {: .error} -> -> This file **must** be owned by `root` and be neither group- nor -> world-writable (e.g. mode `0644`). The setuid helper refuses to start -> otherwise — a present-but-untrusted file is treated as operator -> misconfiguration and is fatal (exit `2`), never silently ignored. +You can configure telemetry with Hyper by adding this section to your +configuration and Hyper will emit tracing spans as configured. -### Root keys +| Config Key | `config.exs` | `config.toml`| Default | Notes | +|------------|-------------------------------------|-------------------------|---------|--------------------------------------------------------------------------------------------------------| +| `proto` | `.proto` | `.proto` | - | | +| `endpoint` | `.endpoint` | `.endpoint` | - | | +| `headers` | `.headers` | `.headers` | - | | -| Key | Type | Default | Meaning | -|-----|------|---------|---------| -| `work_dir` | string (absolute path) | `/srv/hyper` | Root of all node-local runtime state. Every other directory is derived from it. Must be an absolute path. Strongly recommended to sit on an NVMe drive. | +### Budget Configuration (`Hyper.Config.Budget`, `[budget]`) -The following directories are derived from `work_dir` and are **not** -independently configurable: +Hyper allows you to control the absolute maximal budgets that are available to +all VMs on a particular node. -| Path | Purpose | -|------|---------| -| `/jails` | Per-VM chroot directories | -| `/socks` | Per-VM control/gRPC sockets | -| `/scratch` | Per-VM copy-on-write writable layers | -| `/layers` | Read-only image layer store | -| `/redist` | Node-downloaded binaries (`vmlinux`, `umoci`) | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +|--------------------|---------------------------------------|---------------------------|-----------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| +| `mem_max` | `.mem_max` | `.mem_max` | `"hyper"` | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of RAM usage. This value **must not** exceed available system memory. | +| `disk_max` | `.disk_max` | `.disk_max` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of disk usage. This value **must not** exceed available system disk space. | +| `cpu_max_load` | `.cpu_max_load` | `.cpu_max_load` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of CPU usage. | +| `cpu_max_cap` | `.cpu_max_cap` | `.cpu_max_cap` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of CPU usage. | +| `disk_bw_cap` | `.disk_bw_cap` | `.disk_bw_cap` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of disk bandwidth. | +| `disk_bw_max_load` | `.disk_bw_max_load` | `.disk_bw_max_load` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of disk bandwidth. | +| `net_bw_cap` | `.net_bw_cap` | `.net_bw_cap` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of net bandwidth. | +| `net_bw_max_load` | `.net_bw_max_load` | `.net_bw_max_load` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of net bandwidth. | -### `[jails]` — confinement +### VmLinux Paths (`Hyper.Config.VmLinux`, `[vmlinux]`) -| Key | Type | Default | Meaning | -|-----|------|---------|---------| -| `cgroup` | string | `"hyper"` | Parent cgroup under which every VM cgroup is nested (passed to the jailer as `--parent-cgroup`). The operator must create `/sys/fs/cgroup/` and enable subtree control. | -| `uid_gid_range` | `[min, max]` | `[900000, 999999]` | UID/GID band each VM jail is allocated from. `min` must be `>= 1` and `<= max`; `min = 0` is rejected (uid 0 is root, and the jailer skips its privilege drop for uid 0). | +Hyper requires Linux images for the architectures it runs on: -> #### `uid_gid_range` is enforced on both sides {: .warning} -> -> The node only hands out UIDs in this range, and the helper only *accepts* -> UIDs in this range. Because both read the same file, narrowing the band is a -> single edit here — no second place to keep in sync. Nothing else on the host -> may use UIDs/GIDs in this range. - -### Complete example - -```toml -# Root of all node-local state. Strongly prefer an NVMe-backed mount. -work_dir = "/srv/hyper" - -# External binaries. The privileged ones (firecracker..blockdev) must be -# root-owned, not group/world-writable, absolute, and named exactly as their -# key; the node tools (skopeo/mke2fs/umoci/suidhelper) have no such requirement. -[tools] -firecracker = "/opt/firecracker/firecracker" # required; basename must be 'firecracker' -jailer = "/opt/firecracker/jailer" # required; basename must be 'jailer' -# dmsetup = "/usr/sbin/dmsetup" # optional (default shown) -# losetup = "/usr/sbin/losetup" # optional (default shown) -# blockdev = "/usr/sbin/blockdev" # optional (default shown) -# skopeo = "skopeo" # optional node tool (default shown) -# mke2fs = "mke2fs" # optional node tool (default shown) -# umoci = "/usr/bin/umoci" # optional; omit to auto-download -# suidhelper = "/usr/local/bin/hyper-suidhelper" # optional (default shown) - -[jails] -cgroup = "hyper" # default -uid_gid_range = [900000, 999999] # default -``` - -The minimal file is just `work_dir` plus the two required tools — everything -else defaults. - -## Node-only configuration (`config :hyper`) - -These have no helper counterpart and stay in `config :hyper`. The node's tool -paths (`skopeo`, `mke2fs`, `umoci`, `suidhelper`) used to live here but now read -from the `[tools]` table above — see [Tool Configuration](#tool-configuration). - -### Guest kernels - -| Key | Where read | Type | Default | Meaning | -|-----|-----------|------|---------|---------| -| `vmlinux` | runtime | `%{arch => path}` | `%{}` | Per-architecture guest kernel images, keyed by `Sys.Arch.t()`. The operator places kernels on the host and points these at them. | - -```elixir -config :hyper, - vmlinux: %{x86_64: "/srv/hyper/redist/vmlinux/vmlinux-x86_64"} -``` - -### Resource budget — `Hyper.Node.Config.Budget` - -The per-node resource budget. **Required**: the node refuses to boot if it is -absent. Set it in `config/runtime.exs`. Use the `Unit.*` quantities, never bare -numbers. - -| Key | Type | Meaning | -|-----|------|---------| -| `mem_max` | `Unit.Information.t()` | Hard memory cap for this node. | -| `disk_max` | `Unit.Information.t()` | Hard disk cap for this node. | -| `cpu_max_load` | float `0.0..1.0` | CPU-utilization fraction above which the node is considered full. | -| `disk_bw_cap` | `Unit.Bandwidth.t()` | Absolute disk throughput capacity. | -| `disk_bw_max_load` | float `0.0..1.0` | Fraction of `disk_bw_cap` past which disk is saturated. | -| `net_bw_cap` | `Unit.Bandwidth.t()` | Absolute network throughput capacity. | -| `net_bw_max_load` | float `0.0..1.0` | Fraction of `net_bw_cap` past which network is saturated. | - -```elixir -config :hyper, Hyper.Node.Config.Budget, - mem_max: Unit.Information.gib(4), - disk_max: Unit.Information.gib(4), - cpu_max_load: 0.8, - disk_bw_cap: Unit.Bandwidth.gibps(1), - disk_bw_max_load: 0.8, - net_bw_cap: Unit.Bandwidth.gibps(1), - net_bw_max_load: 0.8 -``` - -### gRPC server — `Hyper.Grpc.Config` - -The public gRPC interface. **Disabled by default.** - -| Key | Type | Default | Meaning | -|-----|------|---------|---------| -| `enabled` | boolean | `false` | Whether the server starts. | -| `port` | port number | `50051` | Listen port. | -| `cred` | `GRPC.Credential.t()` \| `nil` | `nil` | TLS credential, or `nil` for plaintext. | -| `adapter_opts` | keyword | `[]` | Forwarded to the server adapter, e.g. `[ip: {0, 0, 0, 0}]`. | - -```elixir -config :hyper, Hyper.Grpc.Config, - enabled: true, - port: 50_051, - cred: GRPC.Credential.new(ssl: [certfile: "/path/cert.pem", keyfile: "/path/key.pem"]) -``` - -> #### Co-located nodes {: .info} -> -> Every node binds `:port`. Running multiple nodes on one host requires giving -> each a distinct port. Build the TLS credential where you load your keys -> (e.g. `config/runtime.exs`); Hyper never reads the filesystem on your behalf. - -### Layer garbage collector — `Hyper.Img.Db.Gc.Config` - -A cluster-wide singleton that prunes unreferenced image layers. Every field has -a default; set only what you change. Durations are `Unit.Time` values, so -overrides belong in `config/runtime.exs`. Set `enabled: false` to never start it. - -| Key | Type | Default | Meaning | -|-----|------|---------|---------| -| `enabled` | boolean | `true` | Run the collector at all. | -| `batch_size` | `pos_integer` | `200` | Rows per keyset page (smaller = finer pause granularity). | -| `batch_pause` | `Unit.Time.t()` | `100ms` | Pause between pages within a sweep. | -| `sweep_interval` | `Unit.Time.t()` | `60s` | Rest between completed sweeps. | -| `acquire_interval` | `Unit.Time.t()` | `5s` | How often a standby retries to become the active singleton. | -| `retry` | `Unit.Time.t()` | `60s` | Backoff when the medium or DB is unavailable. | -| `statement_timeout` | `Unit.Time.t()` | `5s` | Cap on each GC DB statement so it can't pin a backend. | -| `grace_period` | `Unit.Time.t()` | `1h` | Never prune a blob younger than this (protects a row whose file is still being published). | - -```elixir -config :hyper, Hyper.Img.Db.Gc.Config, - enabled: true, - sweep_interval: Unit.Time.s(30), - grace_period: Unit.Time.s(60 * 60) -``` - -### Orphaned-resource reaper — `Hyper.Node.Reaper.Config` - -A per-node sweeper that reclaims orphaned firecracker cgroups and `hyper-rw-*` -device-mapper volumes left behind by unclean BEAM deaths. Uses a two-strike -grace period (an orphan must be seen on two consecutive ticks before it is -reaped). Set `enabled: false` to never start it. - -| Key | Type | Default | Meaning | -|-----|------|---------|---------| -| `enabled` | boolean | `true` | Run the reaper at all. | -| `interval` | `Unit.Time.t()` | `60s` | Rest between reap ticks. | - -```elixir -config :hyper, Hyper.Node.Reaper.Config, - enabled: true, - interval: Unit.Time.s(30) -``` - -### Telemetry (OpenTelemetry) - -Tracing is configured in `config/runtime.exs` from environment variables: - -| Variable | Effect | -|----------|--------| -| `HONEYCOMB_API_KEY` | Export to `https://api.honeycomb.io` with this key. | -| `OTEL_EXPORTER_OTLP_ENDPOINT` | If `HONEYCOMB_API_KEY` is unset, export to this OTLP/HTTP endpoint (e.g. a local Collector), no auth header. | - -If neither is set, tracing is disabled. - -### Database and cluster topology - -The image-metadata database (`Hyper.Img.Db.Repo`, a standard Ecto/PostgreSQL -repo) and the cluster topology (`:libcluster`) are configured in -`config/config.exs` like any Elixir app. PostgreSQL is a required runtime -dependency — the node will not boot without a reachable instance. See -[Installation](install.md) for connection setup. +| Config Key | `config.exs` | `config.toml` | Default | Notes | +|----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `amd64` | `.amd64` | `.amd64` | Automatically downloaded from [hyper-vmlinux](https://github.com/harmont-dev/hyper-vmlinux). | [Absolute Path](#absolute-path). | +| `aarch64`| `.aarch64` | `.aarch64` | Automatically downloaded from [hyper-vmlinux](https://github.com/harmont-dev/hyper-vmlinux). | [Absolute Path](#absolute-path). | + +### Image Configuration (`Hyper.Config.Img`, `[img]`) + +Hyper's image provisioning layer has a large set of configuration flags +enabling you to tweak how you want Hyper to manage images. + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +|----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `store` | `.store` | `.store` | - | [Absolute Path](#absolute-path) to the [layer storage medium](./architecture.md#storage). | + +Additionally, sub-sections are available. + +#### Database Configuration (`Hyper.Config.Img.Db`, `[img.db]`) + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +|----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `database` | `.database` | - | `"hyper"` | | +| `username` | `.username` | - | - | | +| `password` | `.password` | - | - | | +| `hostname` | `.hostname` | - | - | | + +#### Garbage Collector Configuration (`Hyper.Config.Img.Gc`, `[img.gc]`) + +Hyper supports a mechanism to prune unreferenced image layers. Unreferenced +image layers occur when an ungraceful crash happens, resulting in entries in +the layer medium which are not referenced by the database, and, consequently, +unusable. This is always enabled. Since this scans through the whole layer +database, it can have an impact on performance, and tweaking it may be +necessary. + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +|----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `batch_size` | `.batch_size` | `.batch_size` | `"hyper"` | `200` | +| `batch_pause` | `.batch_pause` | `.batch_pause` | `100ms` | | +| `sweep_interval` | `.sweep_interval` | `.sweep_interval` | `60s` | | +| `acquire_interval` | `.acquire_interval` | `.acquire_interval` | `5s` | | +| `retry` | `.retry` | `.retry` | `60s` | | +| `timeout` | `.timeout` | `.timeout` | `5s` | | +| `grace_period` | `.grace_period` | `.grace_period` | `1h` | | + +### Cluster Topology (`Hyper.Cfg.Cluster`, `[cluster]`) + + + +### Key Types + +#### Absolute Path + + - Typed as a string in TOML and elixir. + - Must be given as an absolute path. + +#### Safe Path + + - Is an [Absolute Path](#absolute-path). + - The basename **must** match the configuration, eg. `firecracker` must have + a path `/foo/bar/firecracker`. + - Must be owned by `root`. + - Must be only writable by `root`. + - Must not be a symlink. + +#### Range + + - Typed as a 2-tuple in Elixir + - Typed as an array of two elements in TOML. + - `[min, max]` semantics. + +#### Unit + + - Represents a unit as defined in `Unit.*`. diff --git a/lib/hyper/cfg.ex b/lib/hyper/cfg.ex index e08338ac..13975844 100644 --- a/lib/hyper/cfg.ex +++ b/lib/hyper/cfg.ex @@ -48,6 +48,7 @@ defmodule Hyper.Cfg do @spec resolve([source]) :: {:ok, term()} | :error defp resolve([]), do: :error + defp resolve([source | rest]) do case from(source) do {:ok, value} -> {:ok, value} diff --git a/lib/hyper/cfg/toml.ex b/lib/hyper/cfg/toml.ex index 9ca878c0..b4d20eeb 100644 --- a/lib/hyper/cfg/toml.ex +++ b/lib/hyper/cfg/toml.ex @@ -23,10 +23,6 @@ defmodule Hyper.Cfg.Toml do end) end - @doc "Path to the shared config file." - @spec path :: Path.t() - def path, do: @config_path - @doc "Drop the cache so the next read re-parses the file (test hook)." @spec reload :: map() def reload do diff --git a/lib/hyper/img/db/gc.ex b/lib/hyper/img/db/gc.ex index c16abef8..87588f1a 100644 --- a/lib/hyper/img/db/gc.ex +++ b/lib/hyper/img/db/gc.ex @@ -31,9 +31,9 @@ defmodule Hyper.Img.Db.Gc do require Logger import Ecto.Query + alias Hyper.Cfg.Gc, as: Config alias Hyper.Cluster.Routing alias Hyper.Img.Db.{Blob, ImageLayer, Repo} - alias Hyper.Cfg.Gc, as: Config alias Hyper.Img.Db.Gc.Sweep alias Hyper.Node.Layer.Repo, as: LayerRepo diff --git a/lib/hyper/img/oci_loader.ex b/lib/hyper/img/oci_loader.ex index ab543614..1ba5fb1c 100644 --- a/lib/hyper/img/oci_loader.ex +++ b/lib/hyper/img/oci_loader.ex @@ -185,7 +185,9 @@ defmodule Hyper.Img.OciLoader do defp build_ext4(rootfs, {size, inodes}) do Logger.debug("oci: building #{Information.as_mib(size)} MiB ext4 rootfs (#{inodes} inodes)") File.mkdir_p!(Hyper.Cfg.Dirs.layer_dir()) - staged = Path.join(Hyper.Cfg.Dirs.layer_dir(), ".incoming-#{System.unique_integer([:positive])}.img") + + staged = + Path.join(Hyper.Cfg.Dirs.layer_dir(), ".incoming-#{System.unique_integer([:positive])}.img") args = ["-t", "ext4", "-F", "-q", "-N", to_string(inodes), "-d", rootfs, staged] ++ diff --git a/lib/hyper/node/budget/node_state.ex b/lib/hyper/node/budget/node_state.ex index f80b5e7d..aa9af303 100644 --- a/lib/hyper/node/budget/node_state.ex +++ b/lib/hyper/node/budget/node_state.ex @@ -10,8 +10,8 @@ defmodule Hyper.Node.Budget.NodeState do knowing the target's config or core count. """ - alias Hyper.Node.Budget.Hard alias Hyper.Cfg.Budget, as: Config + alias Hyper.Node.Budget.Hard alias Hyper.Vm.Instance.Spec alias Sys.Mon alias Sys.Mon.Server.Reading diff --git a/lib/hyper/node/img/mutable.ex b/lib/hyper/node/img/mutable.ex index a516ff58..17af34c6 100644 --- a/lib/hyper/node/img/mutable.ex +++ b/lib/hyper/node/img/mutable.ex @@ -149,7 +149,11 @@ defmodule Hyper.Node.Img.Mutable do defp arm_idle(state) do state = cancel_idle(state) - %{state | idle_ref: Process.send_after(self(), :idle_timeout, Hyper.Cfg.Timeouts.idle_ms(:mutable))} + + %{ + state + | idle_ref: Process.send_after(self(), :idle_timeout, Hyper.Cfg.Timeouts.idle_ms(:mutable)) + } end defp cancel_idle(%State{idle_ref: nil} = state), do: state diff --git a/lib/hyper/node/img/server.ex b/lib/hyper/node/img/server.ex index 57364bb5..fff49551 100644 --- a/lib/hyper/node/img/server.ex +++ b/lib/hyper/node/img/server.ex @@ -225,7 +225,11 @@ defmodule Hyper.Node.Img.Server do @spec arm_idle(State.t()) :: State.t() defp arm_idle(state) do state = cancel_idle(state) - %{state | idle_ref: Process.send_after(self(), :idle_timeout, Hyper.Cfg.Timeouts.idle_ms(:img))} + + %{ + state + | idle_ref: Process.send_after(self(), :idle_timeout, Hyper.Cfg.Timeouts.idle_ms(:img)) + } end @spec cancel_idle(State.t()) :: State.t() diff --git a/lib/hyper/node/layer/server.ex b/lib/hyper/node/layer/server.ex index e80848fd..888db658 100644 --- a/lib/hyper/node/layer/server.ex +++ b/lib/hyper/node/layer/server.ex @@ -5,7 +5,8 @@ defmodule Hyper.Node.Layer.Server do Reference-counted via process monitors: each holder `acquire/1`s the layer and the server monitors it, so a holder that crashes is released automatically (no leaked count). When the last holder goes away the server waits a short idle - grace period and then stops, unmounting the block device in `terminate/2`. + grace period and then stops, unmounting the block device in `terminate/2`. The + grace period keeps bursty acquire/release cycles from thrashing the mount. """ use GenServer @@ -148,7 +149,11 @@ defmodule Hyper.Node.Layer.Server do @spec arm_idle(State.t()) :: State.t() defp arm_idle(state) do state = cancel_idle(state) - %{state | idle_ref: Process.send_after(self(), :idle_timeout, Hyper.Cfg.Timeouts.idle_ms(:layer))} + + %{ + state + | idle_ref: Process.send_after(self(), :idle_timeout, Hyper.Cfg.Timeouts.idle_ms(:layer)) + } end @spec cancel_idle(State.t()) :: State.t() diff --git a/test/hyper/cfg/jails_test.exs b/test/hyper/cfg/jails_test.exs index c9791156..1d688298 100644 --- a/test/hyper/cfg/jails_test.exs +++ b/test/hyper/cfg/jails_test.exs @@ -25,5 +25,7 @@ defmodule Hyper.Cfg.JailsTest do assert Jails.range_from([800_000, 899_999]) == {800_000, 899_999} assert Jails.range_from(nil) == {900_000, 999_999} assert Jails.range_from("garbage") == {900_000, 999_999} + # A two-element list of non-integers must fall to the default, never a bogus tuple. + assert Jails.range_from(["a", "b"]) == {900_000, 999_999} end end diff --git a/test/hyper/cfg/tools_test.exs b/test/hyper/cfg/tools_test.exs index 77874593..eed42f1f 100644 --- a/test/hyper/cfg/tools_test.exs +++ b/test/hyper/cfg/tools_test.exs @@ -1,8 +1,8 @@ defmodule Hyper.Cfg.ToolsTest do use ExUnit.Case, async: false - alias Hyper.Cfg.Tools alias Hyper.Cfg.Toml + alias Hyper.Cfg.Tools setup do # Hermetic: empty TOML cache so we assert built-in defaults, not the From a58ff050f0cddf8694db2dbb7f4d2be84b44eed9 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 21:33:45 +0000 Subject: [PATCH 17/47] wrap up config.md --- docs/cookbook/config.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md index 31850efa..4bb18645 100644 --- a/docs/cookbook/config.md +++ b/docs/cookbook/config.md @@ -24,7 +24,7 @@ Note the keys are abbreviated for better layout: - All keys under `config.exs` are written in short-hand form. The parent group is given as the section title. For example, `.mke2fs` in the tool configuration section expands to - `:hyper, Hyper.Config.Tools, mke2fs: "/path/to/mke2fs"`. + `:hyper, Hyper.Cfg.Tools, mke2fs: "/path/to/mke2fs"`. ### Root Keys (`Hyper.Config`, `-`) @@ -32,7 +32,7 @@ Note the keys are abbreviated for better layout: |---------------|-------------------------|--------------------------|-----------------------------------|-------------------------------------------------------------------------| | `work_dir` | - | `work_dir` | - | [Absolute Path](#absolute-path) where `Hyper` creates its working tree. | -### Tool Configuration (`Hyper.Config.Tools`, `[tools]`) +### Tool Configuration (`Hyper.Cfg.Tools`, `[tools]`) Hyper relies on a large number of external tools, of which the paths are configurable: @@ -56,7 +56,7 @@ configurable: | `cgroup` | - | `.cgroup` | `"hyper"` | Parent cgroup under which each VM's cgroup is nested. Each VM receives its own ephemeral cgroup which lives under the umbrella of this cgroup. | | `uid_gid_range`| - | `.uid_gid_range` | - | [Range](#range) limiting the UID/GID values given to VMs. Each VM receives its own UID/GID pair, within these bounds. Must not be an existing user/group. | -### gRPC Configuration (`Hyper.Config.Grpc`, `[grpc]`) +### gRPC Configuration (`Hyper.Cfg.Grpc`, `[grpc]`) Hyper supports a [gRPC](https://grpc.io/) interface enabling you to interface with `Hyper` from any language. @@ -77,7 +77,7 @@ with `Hyper` from any language. > you can also conditionally enable the `gRPC` server based on logic in your > `config.exs`, for example, to only spawn it on your "main" server. -### Telemetry Configuration (`Hyper.Config.Otel`, `[otel]`) +### Telemetry Configuration (`Hyper.Cfg.Otel`, `[otel]`) You can configure telemetry with Hyper by adding this section to your configuration and Hyper will emit tracing spans as configured. @@ -88,7 +88,7 @@ configuration and Hyper will emit tracing spans as configured. | `endpoint` | `.endpoint` | `.endpoint` | - | | | `headers` | `.headers` | `.headers` | - | | -### Budget Configuration (`Hyper.Config.Budget`, `[budget]`) +### Budget Configuration (`Hyper.Cfg.Budget`, `[budget]`) Hyper allows you to control the absolute maximal budgets that are available to all VMs on a particular node. @@ -104,7 +104,7 @@ all VMs on a particular node. | `net_bw_cap` | `.net_bw_cap` | `.net_bw_cap` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of net bandwidth. | | `net_bw_max_load` | `.net_bw_max_load` | `.net_bw_max_load` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of net bandwidth. | -### VmLinux Paths (`Hyper.Config.VmLinux`, `[vmlinux]`) +### VmLinux Paths (`Hyper.Cfg.VmLinux`, `[vmlinux]`) Hyper requires Linux images for the architectures it runs on: @@ -113,7 +113,7 @@ Hyper requires Linux images for the architectures it runs on: | `amd64` | `.amd64` | `.amd64` | Automatically downloaded from [hyper-vmlinux](https://github.com/harmont-dev/hyper-vmlinux). | [Absolute Path](#absolute-path). | | `aarch64`| `.aarch64` | `.aarch64` | Automatically downloaded from [hyper-vmlinux](https://github.com/harmont-dev/hyper-vmlinux). | [Absolute Path](#absolute-path). | -### Image Configuration (`Hyper.Config.Img`, `[img]`) +### Image Configuration (`Hyper.Cfg.Img`, `[img]`) Hyper's image provisioning layer has a large set of configuration flags enabling you to tweak how you want Hyper to manage images. @@ -124,7 +124,7 @@ enabling you to tweak how you want Hyper to manage images. Additionally, sub-sections are available. -#### Database Configuration (`Hyper.Config.Img.Db`, `[img.db]`) +#### Database Configuration (`Hyper.Cfg.Img.Db`, `[img.db]`) | Config Key | `config.exs` | `config.toml` | Default | Notes | |----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -133,7 +133,7 @@ Additionally, sub-sections are available. | `password` | `.password` | - | - | | | `hostname` | `.hostname` | - | - | | -#### Garbage Collector Configuration (`Hyper.Config.Img.Gc`, `[img.gc]`) +#### Garbage Collector Configuration (`Hyper.Cfg.Img.Gc`, `[img.gc]`) Hyper supports a mechanism to prune unreferenced image layers. Unreferenced image layers occur when an ungraceful crash happens, resulting in entries in From b6e00c75a47222713f591f7f906126d830d1d7f8 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 21:34:37 +0000 Subject: [PATCH 18/47] even better docs --- docs/cookbook/config.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md index 4bb18645..ab4d8f59 100644 --- a/docs/cookbook/config.md +++ b/docs/cookbook/config.md @@ -14,7 +14,7 @@ Configuring `Hyper` is done through four layers, in priority: **Note that not all layers allow all configuration fields to be tweaked.** Read further for more details on where and how each configuration field is set. -## Configuration Fields +# Configuration Fields This section briefly outlines the configuration fields available in `Hyper`. Note the keys are abbreviated for better layout: @@ -26,13 +26,13 @@ Note the keys are abbreviated for better layout: configuration section expands to `:hyper, Hyper.Cfg.Tools, mke2fs: "/path/to/mke2fs"`. -### Root Keys (`Hyper.Config`, `-`) +## Root Keys (`Hyper.Config`, `-`) | Config Key | `config.exs` | `config.toml` | Default | Notes | |---------------|-------------------------|--------------------------|-----------------------------------|-------------------------------------------------------------------------| | `work_dir` | - | `work_dir` | - | [Absolute Path](#absolute-path) where `Hyper` creates its working tree. | -### Tool Configuration (`Hyper.Cfg.Tools`, `[tools]`) +## Tool Configuration (`Hyper.Cfg.Tools`, `[tools]`) Hyper relies on a large number of external tools, of which the paths are configurable: @@ -49,14 +49,14 @@ configurable: | `umoci` | `.umoci` | `.umoci` | Automatically downloaded. | | | `suidhelper` | `.suidhelper` | `.suidhelper` | `"/usr/local/bin/hyper-suidhelper"` | [Absolute Path](#absolute-path) | -### Jail Confinement (`-`, `[jails]`) +## Jail Confinement (`-`, `[jails]`) | Config Key | `config.exs` | `config.toml` | Default | Notes | |----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| | `cgroup` | - | `.cgroup` | `"hyper"` | Parent cgroup under which each VM's cgroup is nested. Each VM receives its own ephemeral cgroup which lives under the umbrella of this cgroup. | | `uid_gid_range`| - | `.uid_gid_range` | - | [Range](#range) limiting the UID/GID values given to VMs. Each VM receives its own UID/GID pair, within these bounds. Must not be an existing user/group. | -### gRPC Configuration (`Hyper.Cfg.Grpc`, `[grpc]`) +## gRPC Configuration (`Hyper.Cfg.Grpc`, `[grpc]`) Hyper supports a [gRPC](https://grpc.io/) interface enabling you to interface with `Hyper` from any language. @@ -77,7 +77,7 @@ with `Hyper` from any language. > you can also conditionally enable the `gRPC` server based on logic in your > `config.exs`, for example, to only spawn it on your "main" server. -### Telemetry Configuration (`Hyper.Cfg.Otel`, `[otel]`) +## Telemetry Configuration (`Hyper.Cfg.Otel`, `[otel]`) You can configure telemetry with Hyper by adding this section to your configuration and Hyper will emit tracing spans as configured. @@ -88,7 +88,7 @@ configuration and Hyper will emit tracing spans as configured. | `endpoint` | `.endpoint` | `.endpoint` | - | | | `headers` | `.headers` | `.headers` | - | | -### Budget Configuration (`Hyper.Cfg.Budget`, `[budget]`) +## Budget Configuration (`Hyper.Cfg.Budget`, `[budget]`) Hyper allows you to control the absolute maximal budgets that are available to all VMs on a particular node. @@ -104,7 +104,7 @@ all VMs on a particular node. | `net_bw_cap` | `.net_bw_cap` | `.net_bw_cap` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of net bandwidth. | | `net_bw_max_load` | `.net_bw_max_load` | `.net_bw_max_load` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of net bandwidth. | -### VmLinux Paths (`Hyper.Cfg.VmLinux`, `[vmlinux]`) +## VmLinux Paths (`Hyper.Cfg.VmLinux`, `[vmlinux]`) Hyper requires Linux images for the architectures it runs on: @@ -113,7 +113,7 @@ Hyper requires Linux images for the architectures it runs on: | `amd64` | `.amd64` | `.amd64` | Automatically downloaded from [hyper-vmlinux](https://github.com/harmont-dev/hyper-vmlinux). | [Absolute Path](#absolute-path). | | `aarch64`| `.aarch64` | `.aarch64` | Automatically downloaded from [hyper-vmlinux](https://github.com/harmont-dev/hyper-vmlinux). | [Absolute Path](#absolute-path). | -### Image Configuration (`Hyper.Cfg.Img`, `[img]`) +## Image Configuration (`Hyper.Cfg.Img`, `[img]`) Hyper's image provisioning layer has a large set of configuration flags enabling you to tweak how you want Hyper to manage images. @@ -124,7 +124,7 @@ enabling you to tweak how you want Hyper to manage images. Additionally, sub-sections are available. -#### Database Configuration (`Hyper.Cfg.Img.Db`, `[img.db]`) +### Database Configuration (`Hyper.Cfg.Img.Db`, `[img.db]`) | Config Key | `config.exs` | `config.toml` | Default | Notes | |----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -133,7 +133,7 @@ Additionally, sub-sections are available. | `password` | `.password` | - | - | | | `hostname` | `.hostname` | - | - | | -#### Garbage Collector Configuration (`Hyper.Cfg.Img.Gc`, `[img.gc]`) +### Garbage Collector Configuration (`Hyper.Cfg.Img.Gc`, `[img.gc]`) Hyper supports a mechanism to prune unreferenced image layers. Unreferenced image layers occur when an ungraceful crash happens, resulting in entries in @@ -152,11 +152,11 @@ necessary. | `timeout` | `.timeout` | `.timeout` | `5s` | | | `grace_period` | `.grace_period` | `.grace_period` | `1h` | | -### Cluster Topology (`Hyper.Cfg.Cluster`, `[cluster]`) +## Cluster Topology (`Hyper.Cfg.Cluster`, `[cluster]`) -### Key Types +## Key Types #### Absolute Path From 40da0ee30c9a5fe246baec49a34b9c8e64420dc5 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 21:59:25 +0000 Subject: [PATCH 19/47] feat(cfg): non-raising fetch_cfg/1 resolver sibling --- lib/hyper/cfg.ex | 6 +++++- test/hyper/cfg/resolver_test.exs | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/hyper/cfg.ex b/lib/hyper/cfg.ex index 13975844..b41ece70 100644 --- a/lib/hyper/cfg.ex +++ b/lib/hyper/cfg.ex @@ -31,10 +31,14 @@ defmodule Hyper.Cfg do | {:toml, String.t()} | {:default, term()} + @doc false + @spec fetch_cfg([source]) :: {:ok, term()} | :error + def fetch_cfg(sources) when is_list(sources), do: resolve(sources) + @doc false @spec get_cfg([source]) :: term() def get_cfg(sources) when is_list(sources) do - case resolve(sources) do + case fetch_cfg(sources) do {:ok, value} -> value diff --git a/test/hyper/cfg/resolver_test.exs b/test/hyper/cfg/resolver_test.exs index 0752ce54..41cabbea 100644 --- a/test/hyper/cfg/resolver_test.exs +++ b/test/hyper/cfg/resolver_test.exs @@ -33,4 +33,13 @@ defmodule Hyper.Cfg.ResolverTest do get_cfg(toml: "definitely.absent") end end + + test "fetch_cfg/1 returns {:ok, value} or :error without raising" do + Application.put_env(:hyper, :__cfg_test, "v") + assert Hyper.Cfg.fetch_cfg(runtime: :__cfg_test) == {:ok, "v"} + Application.delete_env(:hyper, :__cfg_test) + assert Hyper.Cfg.fetch_cfg(runtime: :__cfg_test) == :error + assert Hyper.Cfg.fetch_cfg(toml: "definitely.absent") == :error + assert Hyper.Cfg.fetch_cfg(runtime: :__cfg_test, default: "d") == {:ok, "d"} + end end From b9e2fbe98e0a55ec2a95bec67042f1cecc1a7c08 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 22:02:56 +0000 Subject: [PATCH 20/47] feat(unit): string parsers for Information/Bandwidth/Time --- lib/unit/bandwidth.ex | 20 +++++++++++++++++++ lib/unit/information.ex | 20 +++++++++++++++++++ lib/unit/time.ex | 27 ++++++++++++++++++++++++++ test/unit/bandwidth_parse_test.exs | 25 ++++++++++++++++++++++++ test/unit/information_parse_test.exs | 29 ++++++++++++++++++++++++++++ test/unit/time_parse_test.exs | 28 +++++++++++++++++++++++++++ 6 files changed, 149 insertions(+) create mode 100644 test/unit/bandwidth_parse_test.exs create mode 100644 test/unit/information_parse_test.exs create mode 100644 test/unit/time_parse_test.exs diff --git a/lib/unit/bandwidth.ex b/lib/unit/bandwidth.ex index 5811d444..7a158952 100644 --- a/lib/unit/bandwidth.ex +++ b/lib/unit/bandwidth.ex @@ -37,6 +37,26 @@ defmodule Unit.Bandwidth do @doc "The zero throughput (additive identity)." @spec zero() :: t() def zero, do: %__MODULE__{bytes_per_sec: 0} + + @units %{"Bps" => 1, "KiBps" => @kib, "MiBps" => @mib, "GiBps" => @gib, "TiBps" => @tib} + + @doc "Parse a string like `\"1GiBps\"`. Suffixes: Bps/KiBps/MiBps/GiBps/TiBps." + @spec parse(String.t()) :: {:ok, t()} | {:error, {:bad_unit, String.t()}} + def parse(s) when is_binary(s) do + case Regex.run(~r/^\s*(\d+)\s*(Bps|KiBps|MiBps|GiBps|TiBps)\s*$/, s) do + [_, n, suffix] -> {:ok, %__MODULE__{bytes_per_sec: String.to_integer(n) * Map.fetch!(@units, suffix)}} + _ -> {:error, {:bad_unit, s}} + end + end + + @doc "Like `parse/1` but raises `ArgumentError` on bad input." + @spec parse!(String.t()) :: t() + def parse!(s) do + case parse(s) do + {:ok, v} -> v + {:error, _} -> raise ArgumentError, "invalid Bandwidth string: #{inspect(s)}" + end + end end defimpl Unit.Quantity, for: Unit.Bandwidth do diff --git a/lib/unit/information.ex b/lib/unit/information.ex index be9a31a0..4e79b1cf 100644 --- a/lib/unit/information.ex +++ b/lib/unit/information.ex @@ -43,6 +43,26 @@ defmodule Unit.Information do @doc "The zero quantity (additive identity)." @spec zero() :: t() def zero, do: %__MODULE__{bytes: 0} + + @units %{"B" => 1, "KiB" => @kib, "MiB" => @mib, "GiB" => @gib, "TiB" => @tib} + + @doc "Parse a string like `\"4GiB\"` into an `Information`. Suffixes: B/KiB/MiB/GiB/TiB." + @spec parse(String.t()) :: {:ok, t()} | {:error, {:bad_unit, String.t()}} + def parse(s) when is_binary(s) do + case Regex.run(~r/^\s*(\d+)\s*(B|KiB|MiB|GiB|TiB)\s*$/, s) do + [_, n, suffix] -> {:ok, %__MODULE__{bytes: String.to_integer(n) * Map.fetch!(@units, suffix)}} + _ -> {:error, {:bad_unit, s}} + end + end + + @doc "Like `parse/1` but raises `ArgumentError` on bad input." + @spec parse!(String.t()) :: t() + def parse!(s) do + case parse(s) do + {:ok, v} -> v + {:error, _} -> raise ArgumentError, "invalid Information string: #{inspect(s)}" + end + end end defimpl Unit.Quantity, for: Unit.Information do diff --git a/lib/unit/time.ex b/lib/unit/time.ex index b15950e5..5f7f5f01 100644 --- a/lib/unit/time.ex +++ b/lib/unit/time.ex @@ -42,6 +42,33 @@ defmodule Unit.Time do @doc "The zero duration (additive identity)." @spec zero() :: t() def zero, do: %__MODULE__{ns: 0} + + @units %{ + "ns" => 1, + "us" => @us, + "ms" => @ms, + "s" => @s, + "m" => 60 * @s, + "h" => 3600 * @s + } + + @doc "Parse a duration string like `\"60s\"`/`\"100ms\"`/`\"1h\"`. Suffixes: ns/us/ms/s/m/h." + @spec parse(String.t()) :: {:ok, t()} | {:error, {:bad_unit, String.t()}} + def parse(s) when is_binary(s) do + case Regex.run(~r/^\s*(\d+)\s*(ns|us|ms|s|m|h)\s*$/, s) do + [_, n, suffix] -> {:ok, %__MODULE__{ns: String.to_integer(n) * Map.fetch!(@units, suffix)}} + _ -> {:error, {:bad_unit, s}} + end + end + + @doc "Like `parse/1` but raises `ArgumentError` on bad input." + @spec parse!(String.t()) :: t() + def parse!(s) do + case parse(s) do + {:ok, v} -> v + {:error, _} -> raise ArgumentError, "invalid Time string: #{inspect(s)}" + end + end end defimpl Unit.Quantity, for: Unit.Time do diff --git a/test/unit/bandwidth_parse_test.exs b/test/unit/bandwidth_parse_test.exs new file mode 100644 index 00000000..e9154ddd --- /dev/null +++ b/test/unit/bandwidth_parse_test.exs @@ -0,0 +1,25 @@ +defmodule Unit.BandwidthParseTest do + use ExUnit.Case, async: true + use ExUnitProperties + + alias Unit.Bandwidth + + test "parses each suffix" do + assert Bandwidth.parse!("100Bps") == Bandwidth.bps(100) + assert Bandwidth.parse!("4KiBps") == Bandwidth.kibps(4) + assert Bandwidth.parse!("512MiBps") == Bandwidth.mibps(512) + assert Bandwidth.parse!("1GiBps") == Bandwidth.gibps(1) + assert Bandwidth.parse!("1 GiBps") == Bandwidth.gibps(1) + end + + test "rejects garbage" do + assert {:error, _} = Bandwidth.parse("1GiB") + assert_raise ArgumentError, fn -> Bandwidth.parse!("fast") end + end + + property "parse! inverts gibps" do + check all n <- integer(0..1024) do + assert Bandwidth.parse!("#{n}GiBps") == Bandwidth.gibps(n) + end + end +end diff --git a/test/unit/information_parse_test.exs b/test/unit/information_parse_test.exs new file mode 100644 index 00000000..ce8f7fde --- /dev/null +++ b/test/unit/information_parse_test.exs @@ -0,0 +1,29 @@ +defmodule Unit.InformationParseTest do + use ExUnit.Case, async: true + use ExUnitProperties + + alias Unit.Information + + test "parses each suffix to the right magnitude" do + assert Information.parse!("100B") == Information.bytes(100) + assert Information.parse!("4KiB") == Information.kib(4) + assert Information.parse!("512MiB") == Information.mib(512) + assert Information.parse!("4GiB") == Information.gib(4) + assert Information.parse!("2TiB") == Information.tib(2) + assert Information.parse!("4 GiB") == Information.gib(4) + end + + test "rejects garbage" do + assert {:error, _} = Information.parse("") + assert {:error, _} = Information.parse("GiB") + assert {:error, _} = Information.parse("4 Gigs") + assert {:error, _} = Information.parse("4.5GiB") + assert_raise ArgumentError, fn -> Information.parse!("nope") end + end + + property "parse! inverts the gib constructor" do + check all n <- integer(0..4096) do + assert Information.parse!("#{n}GiB") == Information.gib(n) + end + end +end diff --git a/test/unit/time_parse_test.exs b/test/unit/time_parse_test.exs new file mode 100644 index 00000000..4c5c35e1 --- /dev/null +++ b/test/unit/time_parse_test.exs @@ -0,0 +1,28 @@ +defmodule Unit.TimeParseTest do + use ExUnit.Case, async: true + use ExUnitProperties + + alias Unit.Time + + test "parses each suffix" do + assert Time.parse!("500ns") == Time.ns(500) + assert Time.parse!("100us") == Time.us(100) + assert Time.parse!("100ms") == Time.ms(100) + assert Time.parse!("60s") == Time.s(60) + assert Time.parse!("30m") == Time.s(30 * 60) + assert Time.parse!("1h") == Time.s(3600) + assert Time.parse!("1 h") == Time.s(3600) + end + + test "rejects garbage" do + assert {:error, _} = Time.parse("5") + assert {:error, _} = Time.parse("5 secs") + assert_raise ArgumentError, fn -> Time.parse!("soon") end + end + + property "parse! inverts s" do + check all n <- integer(0..100_000) do + assert Time.parse!("#{n}s") == Time.s(n) + end + end +end From d5d060fc7b7828ed1110d8d169adb672801590d3 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 22:08:43 +0000 Subject: [PATCH 21/47] feat(cfg): dual-source Budget (config.exs + [budget] toml) + cpu_max_cap --- lib/hyper/cfg/budget.ex | 112 +++++++++++++++++++++++---------- test/hyper/cfg/budget_test.exs | 74 ++++++++++++++++++++++ 2 files changed, 152 insertions(+), 34 deletions(-) create mode 100644 test/hyper/cfg/budget_test.exs diff --git a/lib/hyper/cfg/budget.ex b/lib/hyper/cfg/budget.ex index fdcacaae..7ef0c911 100644 --- a/lib/hyper/cfg/budget.ex +++ b/lib/hyper/cfg/budget.ex @@ -1,25 +1,18 @@ defmodule Hyper.Cfg.Budget do @moduledoc """ - This node's resource budget configuration. - - Carries both the hard caps consumed by `Hyper.Node.Budget.Hard` (`mem_max`, - `disk_max`) and the soft load ceilings consumed by `Hyper.Node.Budget.Soft`. - - The soft side names, per metric, the machine's absolute capacity and the - fraction of it past which the node is considered too loaded to take on more - work: - - * `cpu_max_load` - utilization fraction (`0.0..1.0`) above which CPU is full; - CPU capacity is the whole machine, so no separate cap is needed. - * `disk_bw_cap` / `disk_bw_max_load` - absolute disk throughput and the - fraction of it usable before disk is considered saturated. - * `net_bw_cap` / `net_bw_max_load` - the same for network throughput. + This node's resource budget. Each field reads from `config.exs` + (`config :hyper, Hyper.Cfg.Budget, ...`), then the `[budget]` table in + `/etc/hyper/config.toml`, then its default. `Unit.*` quantities may be given + as Elixir terms in `config.exs` or as strings (`"4GiB"`, `"1GiBps"`) in TOML. """ + import Hyper.Cfg, only: [fetch_cfg: 1] + @type t :: %__MODULE__{ mem_max: Unit.Information.t(), disk_max: Unit.Information.t(), cpu_max_load: float(), + cpu_max_cap: float() | nil, disk_bw_cap: Unit.Bandwidth.t(), disk_bw_max_load: float(), net_bw_cap: Unit.Bandwidth.t(), @@ -29,6 +22,7 @@ defmodule Hyper.Cfg.Budget do :mem_max, :disk_max, :cpu_max_load, + :cpu_max_cap, :disk_bw_cap, :disk_bw_max_load, :net_bw_cap, @@ -37,31 +31,81 @@ defmodule Hyper.Cfg.Budget do @spec load :: {:ok, t()} | {:error, term()} def load do - case Application.fetch_env(:hyper, __MODULE__) do - {:ok, env} -> - case safe_struct(env) do - {:ok, config} -> - # TODO(markovejnovic): Check whether the limits are under the - # system limits by querying Sys. - :persistent_term.put(__MODULE__, config) - {:ok, config} + with {:ok, mem_max} <- information(:mem_max, "budget.mem_max"), + {:ok, disk_max} <- information(:disk_max, "budget.disk_max"), + {:ok, cpu_max_load} <- number(:cpu_max_load, "budget.cpu_max_load"), + {:ok, disk_bw_cap} <- bandwidth(:disk_bw_cap, "budget.disk_bw_cap"), + {:ok, disk_bw_max_load} <- number(:disk_bw_max_load, "budget.disk_bw_max_load"), + {:ok, net_bw_cap} <- bandwidth(:net_bw_cap, "budget.net_bw_cap"), + {:ok, net_bw_max_load} <- number(:net_bw_max_load, "budget.net_bw_max_load") do + config = %__MODULE__{ + mem_max: mem_max, + disk_max: disk_max, + cpu_max_load: cpu_max_load, + cpu_max_cap: optional_number(:cpu_max_cap, "budget.cpu_max_cap"), + disk_bw_cap: disk_bw_cap, + disk_bw_max_load: disk_bw_max_load, + net_bw_cap: net_bw_cap, + net_bw_max_load: net_bw_max_load + } + + :persistent_term.put(__MODULE__, config) + {:ok, config} + end + end - {:error, _} = err -> - err - end + @spec get :: t() + def get, do: :persistent_term.get(__MODULE__) - :error -> - {:error, :config_missing} + @spec information(atom(), String.t()) :: {:ok, Unit.Information.t()} | {:error, term()} + defp information(key, toml) do + with {:ok, v} <- required(key, toml) do + coerce(v, &Unit.Information.parse/1, Unit.Information, key) end end - defp safe_struct(env) do - {:ok, struct!(__MODULE__, env)} - rescue - e in KeyError -> {:error, {:unknown_key, e.key}} - e in ArgumentError -> {:error, {:invalid_config, e.message}} + @spec bandwidth(atom(), String.t()) :: {:ok, Unit.Bandwidth.t()} | {:error, term()} + defp bandwidth(key, toml) do + with {:ok, v} <- required(key, toml) do + coerce(v, &Unit.Bandwidth.parse/1, Unit.Bandwidth, key) + end end - @spec get :: t() - def get, do: :persistent_term.get(__MODULE__) + @spec number(atom(), String.t()) :: {:ok, number()} | {:error, term()} + defp number(key, toml) do + case required(key, toml) do + {:ok, n} when is_number(n) -> {:ok, n} + {:ok, other} -> {:error, {:not_a_number, key, other}} + {:error, _} = e -> e + end + end + + @spec optional_number(atom(), String.t()) :: number() | nil + defp optional_number(key, toml) do + case fetch_cfg(runtime: {__MODULE__, key}, toml: toml) do + {:ok, n} when is_number(n) -> n + _ -> nil + end + end + + @spec required(atom(), String.t()) :: {:ok, term()} | {:error, term()} + defp required(key, toml) do + case fetch_cfg(runtime: {__MODULE__, key}, toml: toml) do + {:ok, v} -> {:ok, v} + :error -> {:error, {:missing, key}} + end + end + + # Accept an already-typed struct (from config.exs) or a string to parse (from TOML). + @spec coerce(term(), (String.t() -> {:ok, struct()} | {:error, term()}), module(), atom()) :: + {:ok, struct()} | {:error, term()} + defp coerce(%mod{} = v, _parse, mod, _key), do: {:ok, v} + defp coerce(s, parse, _mod, key) when is_binary(s) do + case parse.(s) do + {:ok, v} -> {:ok, v} + {:error, _} -> {:error, {:bad_value, key, s}} + end + end + + defp coerce(other, _parse, _mod, key), do: {:error, {:bad_value, key, other}} end diff --git a/test/hyper/cfg/budget_test.exs b/test/hyper/cfg/budget_test.exs new file mode 100644 index 00000000..452cc993 --- /dev/null +++ b/test/hyper/cfg/budget_test.exs @@ -0,0 +1,74 @@ +defmodule Hyper.Cfg.BudgetTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.Budget + alias Hyper.Cfg.Toml + + setup do + Application.delete_env(:hyper, Budget) + Toml.put_cache(%{}) + on_exit(fn -> + Application.delete_env(:hyper, Budget) + Toml.reload() + end) + :ok + end + + test "loads Unit values from config.exs terms" do + Application.put_env(:hyper, Budget, + mem_max: Unit.Information.gib(4), + disk_max: Unit.Information.gib(64), + cpu_max_load: 0.8, + disk_bw_cap: Unit.Bandwidth.gibps(1), + disk_bw_max_load: 0.8, + net_bw_cap: Unit.Bandwidth.gibps(1), + net_bw_max_load: 0.8 + ) + + assert {:ok, cfg} = Budget.load() + assert cfg.mem_max == Unit.Information.gib(4) + assert cfg.net_bw_cap == Unit.Bandwidth.gibps(1) + end + + test "loads the same values from a [budget] toml table as strings" do + Toml.put_cache(%{ + "budget" => %{ + "mem_max" => "4GiB", + "disk_max" => "64GiB", + "cpu_max_load" => 0.8, + "cpu_max_cap" => 4.0, + "disk_bw_cap" => "1GiBps", + "disk_bw_max_load" => 0.8, + "net_bw_cap" => "1GiBps", + "net_bw_max_load" => 0.8 + } + }) + + assert {:ok, cfg} = Budget.load() + assert cfg.mem_max == Unit.Information.gib(4) + assert cfg.disk_bw_cap == Unit.Bandwidth.gibps(1) + assert cfg.cpu_max_cap == 4.0 + end + + test "config.exs wins over the toml table" do + Toml.put_cache(%{ + "budget" => %{ + "mem_max" => "1GiB", + "disk_max" => "64GiB", + "cpu_max_load" => 0.8, + "disk_bw_cap" => "1GiBps", + "disk_bw_max_load" => 0.8, + "net_bw_cap" => "1GiBps", + "net_bw_max_load" => 0.8 + } + }) + + Application.put_env(:hyper, Budget, mem_max: Unit.Information.gib(8)) + {:ok, cfg} = Budget.load() + assert cfg.mem_max == Unit.Information.gib(8) + end + + test "a missing required field is an error, not a crash" do + assert {:error, _} = Budget.load() + end +end From 87a6bb4531a8609ab279a7642ea66291f28409aa Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 22:25:29 +0000 Subject: [PATCH 22/47] feat(cfg): dual-source Gc ([img.gc] toml) + rename statement_timeout->timeout --- lib/hyper/cfg/gc.ex | 75 ++++++++++++++++++-------------------- lib/hyper/img/db/gc.ex | 8 ++-- test/hyper/cfg/gc_test.exs | 39 ++++++++++++++++++++ 3 files changed, 79 insertions(+), 43 deletions(-) create mode 100644 test/hyper/cfg/gc_test.exs diff --git a/lib/hyper/cfg/gc.ex b/lib/hyper/cfg/gc.ex index 86f572f6..f44539d3 100644 --- a/lib/hyper/cfg/gc.ex +++ b/lib/hyper/cfg/gc.ex @@ -1,32 +1,13 @@ defmodule Hyper.Cfg.Gc do @moduledoc """ - Configuration for the layer garbage collector (`Hyper.Img.Db.Gc`). - - Every field has a default, so configuration is optional - set only what you want - to change. Durations are `Unit.Time` values, so (like `Hyper.Cfg.Budget`) - overrides belong in `config/runtime.exs`: - - config :hyper, Hyper.Cfg.Gc, - enabled: true, - sweep_interval: Unit.Time.s(30), - grace_period: Unit.Time.s(60 * 60) - - Set `enabled: false` to turn the collector off entirely - it then never starts. - - ## Fields - - * `enabled` - run the collector at all (default `true`). - * `batch_size` - rows per keyset page (default `200`). - * `batch_pause` - pause between pages within a sweep (default `100ms`). - * `sweep_interval` - rest between completed sweeps (default `60s`). - * `acquire_interval` - how often a standby retries to become active (default `5s`). - * `retry` - backoff after the medium or database is unavailable (default `60s`). - * `statement_timeout` - cap on each GC DB statement so it can't pin a backend - (default `5s`). - * `grace_period` - never prune a blob younger than this, so a row whose file is - still being published is safe (default `1h`). + Layer garbage collector tuning. Each field reads from `config.exs` + (`config :hyper, Hyper.Cfg.Gc, ...`), then the `[img.gc]` table, then its + default. Durations are `Unit.Time` — Elixir terms in `config.exs`, strings + (`"60s"`, `"1h"`) in TOML. """ + import Hyper.Cfg, only: [get_cfg: 1] + @type t :: %__MODULE__{ enabled: boolean(), batch_size: pos_integer(), @@ -34,22 +15,38 @@ defmodule Hyper.Cfg.Gc do sweep_interval: Unit.Time.t(), acquire_interval: Unit.Time.t(), retry: Unit.Time.t(), - statement_timeout: Unit.Time.t(), + timeout: Unit.Time.t(), grace_period: Unit.Time.t() } - - defstruct enabled: true, - batch_size: 200, - batch_pause: Unit.Time.ms(100), - sweep_interval: Unit.Time.s(60), - acquire_interval: Unit.Time.s(5), - retry: Unit.Time.s(60), - statement_timeout: Unit.Time.s(5), - grace_period: Unit.Time.s(60 * 60) - - @doc "Build the config from app env, filling any unset field with its default." - @spec load() :: t() + defstruct [ + :enabled, + :batch_size, + :batch_pause, + :sweep_interval, + :acquire_interval, + :retry, + :timeout, + :grace_period + ] + + @spec load :: t() def load do - struct!(__MODULE__, Application.get_env(:hyper, __MODULE__, [])) + struct!(__MODULE__, [ + {:enabled, get_cfg(runtime: {__MODULE__, :enabled}, toml: "img.gc.enabled", default: true)}, + {:batch_size, get_cfg(runtime: {__MODULE__, :batch_size}, toml: "img.gc.batch_size", default: 200)}, + {:batch_pause, duration(:batch_pause, "img.gc.batch_pause", Unit.Time.ms(100))}, + {:sweep_interval, duration(:sweep_interval, "img.gc.sweep_interval", Unit.Time.s(60))}, + {:acquire_interval, duration(:acquire_interval, "img.gc.acquire_interval", Unit.Time.s(5))}, + {:retry, duration(:retry, "img.gc.retry", Unit.Time.s(60))}, + {:timeout, duration(:timeout, "img.gc.timeout", Unit.Time.s(5))}, + {:grace_period, duration(:grace_period, "img.gc.grace_period", Unit.Time.s(3600))} + ]) + end + + defp duration(key, toml, default) do + case get_cfg(runtime: {__MODULE__, key}, toml: toml, default: default) do + %_mod{} = t -> t + s when is_binary(s) -> Unit.Time.parse!(s) + end end end diff --git a/lib/hyper/img/db/gc.ex b/lib/hyper/img/db/gc.ex index 87588f1a..a02b70de 100644 --- a/lib/hyper/img/db/gc.ex +++ b/lib/hyper/img/db/gc.ex @@ -100,7 +100,7 @@ defmodule Hyper.Img.Db.Gc do try do {:noreply, scan_one_batch(state)} rescue - # Only swallow database unavailability (incl. statement_timeout aborts) + # Only swallow database unavailability (incl. timed-out statements) # and retry; let any other exception crash so a real bug surfaces. e in [Postgrex.Error, DBConnection.ConnectionError] -> Logger.warning( @@ -248,11 +248,11 @@ defmodule Hyper.Img.Db.Gc do state |> with_low_priority(fn -> Repo.all(query) end) |> MapSet.new() end - # Run a DB operation at low priority: in a transaction whose statement_timeout - # is capped, so it can never pin a backend and yields under contention. + # Run a DB operation at low priority: in a transaction with a capped per-statement + # timeout, so it can never pin a backend and yields under contention. @spec with_low_priority(t(), (-> result)) :: result when result: var defp with_low_priority(state, fun) do - timeout = Unit.Time.as_ms(state.config.statement_timeout) + timeout = Unit.Time.as_ms(state.config.timeout) {:ok, result} = Repo.transaction(fn -> diff --git a/test/hyper/cfg/gc_test.exs b/test/hyper/cfg/gc_test.exs new file mode 100644 index 00000000..4dd23419 --- /dev/null +++ b/test/hyper/cfg/gc_test.exs @@ -0,0 +1,39 @@ +defmodule Hyper.Cfg.GcTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.Gc + alias Hyper.Cfg.Toml + + setup do + Application.delete_env(:hyper, Gc) + Toml.put_cache(%{}) + on_exit(fn -> + Application.delete_env(:hyper, Gc) + Toml.reload() + end) + :ok + end + + test "defaults when nothing configured" do + cfg = Gc.load() + assert cfg.enabled == true + assert cfg.batch_size == 200 + assert cfg.sweep_interval == Unit.Time.s(60) + assert cfg.timeout == Unit.Time.s(5) + assert cfg.grace_period == Unit.Time.s(3600) + end + + test "reads durations from [img.gc] toml as strings" do + Toml.put_cache(%{"img" => %{"gc" => %{"sweep_interval" => "30s", "grace_period" => "1h", "batch_size" => 50}}}) + cfg = Gc.load() + assert cfg.sweep_interval == Unit.Time.s(30) + assert cfg.grace_period == Unit.Time.s(3600) + assert cfg.batch_size == 50 + end + + test "config.exs Unit term wins over toml string" do + Toml.put_cache(%{"img" => %{"gc" => %{"sweep_interval" => "30s"}}}) + Application.put_env(:hyper, Gc, sweep_interval: Unit.Time.s(90)) + assert Gc.load().sweep_interval == Unit.Time.s(90) + end +end From 7e821490846a13ec23c439d8eab6e9e32744a2bc Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 22:29:35 +0000 Subject: [PATCH 23/47] feat(cfg): dual-source Grpc ([grpc] toml) + cred coercion --- lib/hyper/cfg/grpc.ex | 19 ++++++++++++++-- test/hyper/cfg/grpc_test.exs | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 test/hyper/cfg/grpc_test.exs diff --git a/lib/hyper/cfg/grpc.ex b/lib/hyper/cfg/grpc.ex index 6210512f..68943f67 100644 --- a/lib/hyper/cfg/grpc.ex +++ b/lib/hyper/cfg/grpc.ex @@ -36,9 +36,24 @@ defmodule Hyper.Cfg.Grpc do adapter_opts: keyword() } - @doc "Load the gRPC server configuration from application env." + import Hyper.Cfg, only: [get_cfg: 1] + + @doc "Load the gRPC server configuration: config.exs > [grpc] toml > defaults." @spec load() :: t() - def load, do: struct!(__MODULE__, Application.get_env(:hyper, __MODULE__, [])) + def load do + %__MODULE__{ + enabled: get_cfg(runtime: {__MODULE__, :enabled}, toml: "grpc.enabled", default: false), + port: get_cfg(runtime: {__MODULE__, :port}, toml: "grpc.port", default: @default_port), + cred: cred(get_cfg(runtime: {__MODULE__, :cred}, toml: "grpc.cred", default: nil)), + adapter_opts: get_cfg(runtime: {__MODULE__, :adapter_opts}, toml: "grpc.adapter_opts", default: []) + } + end + + @spec cred(term()) :: GRPC.Credential.t() | nil + defp cred(nil), do: nil + defp cred(%GRPC.Credential{} = c), do: c + defp cred(%{"cert" => cert, "key" => key}), + do: GRPC.Credential.new(ssl: [certfile: cert, keyfile: key]) @doc """ The `GRPC.Server.Supervisor` options for this config: the endpoint and port, diff --git a/test/hyper/cfg/grpc_test.exs b/test/hyper/cfg/grpc_test.exs new file mode 100644 index 00000000..cf241a8c --- /dev/null +++ b/test/hyper/cfg/grpc_test.exs @@ -0,0 +1,42 @@ +defmodule Hyper.Cfg.GrpcTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.Grpc + alias Hyper.Cfg.Toml + + setup do + Application.delete_env(:hyper, Grpc) + Toml.put_cache(%{}) + on_exit(fn -> + Application.delete_env(:hyper, Grpc) + Toml.reload() + end) + :ok + end + + test "defaults: disabled on 50051, no cred" do + cfg = Grpc.load() + assert cfg.enabled == false + assert cfg.port == 50_051 + assert cfg.cred == nil + end + + test "reads enabled/port from [grpc] toml" do + Toml.put_cache(%{"grpc" => %{"enabled" => true, "port" => 6000}}) + cfg = Grpc.load() + assert cfg.enabled == true + assert cfg.port == 6000 + end + + test "builds a credential from a toml inline table" do + Toml.put_cache(%{"grpc" => %{"cred" => %{"cert" => "/c.pem", "key" => "/k.pem"}}}) + cfg = Grpc.load() + assert match?(%GRPC.Credential{}, cfg.cred) + end + + test "config.exs wins over toml" do + Toml.put_cache(%{"grpc" => %{"port" => 6000}}) + Application.put_env(:hyper, Grpc, port: 7000) + assert Grpc.load().port == 7000 + end +end From cefac86df963659752649847ade8acc6ff8b4437 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 22:33:32 +0000 Subject: [PATCH 24/47] feat(cfg): reshape VmLinux to amd64/aarch64 dual-source keys --- lib/hyper/cfg/vm_linux.ex | 21 +++++++++++++++++++++ lib/hyper/cfg/vmlinux.ex | 14 -------------- lib/hyper/node/vmlinux.ex | 6 +++--- test/hyper/cfg/vm_linux_test.exs | 31 +++++++++++++++++++++++++++++++ test/hyper/cfg/vmlinux_test.exs | 13 ------------- 5 files changed, 55 insertions(+), 30 deletions(-) create mode 100644 lib/hyper/cfg/vm_linux.ex delete mode 100644 lib/hyper/cfg/vmlinux.ex create mode 100644 test/hyper/cfg/vm_linux_test.exs delete mode 100644 test/hyper/cfg/vmlinux_test.exs diff --git a/lib/hyper/cfg/vm_linux.ex b/lib/hyper/cfg/vm_linux.ex new file mode 100644 index 00000000..0b3eadbd --- /dev/null +++ b/lib/hyper/cfg/vm_linux.ex @@ -0,0 +1,21 @@ +defmodule Hyper.Cfg.VmLinux do + @moduledoc """ + Per-architecture guest-kernel image paths. Operators set `amd64`/`aarch64` + (mapped to `:x86_64`/`:aarch64`) in `config :hyper, Hyper.Cfg.VmLinux, ...` or + the `[vmlinux]` toml table. An unset architecture is simply absent from the map. + """ + + import Hyper.Cfg, only: [fetch_cfg: 1] + + @archs %{amd64: :x86_64, aarch64: :aarch64} + + @doc "Resolved `%{arch => path}` kernel map (config.exs per key > [vmlinux] toml)." + @spec images :: %{optional(Sys.Arch.t()) => Path.t()} + def images do + for {doc_key, arch} <- @archs, {:ok, path} <- [resolve(doc_key)], into: %{}, do: {arch, path} + end + + @spec resolve(atom()) :: {:ok, Path.t()} | :error + defp resolve(doc_key), + do: fetch_cfg(runtime: {__MODULE__, doc_key}, toml: "vmlinux.#{doc_key}") +end diff --git a/lib/hyper/cfg/vmlinux.ex b/lib/hyper/cfg/vmlinux.ex deleted file mode 100644 index 3b4ee8be..00000000 --- a/lib/hyper/cfg/vmlinux.ex +++ /dev/null @@ -1,14 +0,0 @@ -defmodule Hyper.Cfg.Vmlinux do - @moduledoc """ - Per-architecture guest-kernel image paths, set by the operator in - `config :hyper, vmlinux: %{arch => path}`. Node-only; no helper counterpart. - """ - - import Hyper.Cfg, only: [get_cfg: 1] - - @doc "Operator-configured `%{arch => path}` kernel map, default `%{}`." - # Runtime read, not compile_env: an unset map would inline a literal `%{}`, - # which the type checker proves makes every Map.fetch/2 on it return :error. - @spec images :: %{optional(Sys.Arch.t()) => Path.t()} - def images, do: get_cfg(runtime: :vmlinux, default: %{}) -end diff --git a/lib/hyper/node/vmlinux.ex b/lib/hyper/node/vmlinux.ex index 8742a258..027f0a6b 100644 --- a/lib/hyper/node/vmlinux.ex +++ b/lib/hyper/node/vmlinux.ex @@ -5,7 +5,7 @@ defmodule Hyper.Node.Vmlinux do Two sources, in priority order: 1. An operator-configured path for the node's architecture, via - `config :hyper, vmlinux: %{ => }` (see `Hyper.Cfg.Vmlinux.images/0`). + `config :hyper, Hyper.Cfg.VmLinux, amd64: ..., aarch64: ...` (see `Hyper.Cfg.VmLinux.images/0`). If set, it wins - the operator can pin a custom kernel. 2. Otherwise, the default kernel downloaded by `Hyper.Node.FireVMM.VmLinux.Provider` (highest version for the arch). @@ -24,7 +24,7 @@ defmodule Hyper.Node.Vmlinux do """ @spec path(Sys.Arch.t()) :: Path.t() def path(arch) do - case Map.fetch(Hyper.Cfg.Vmlinux.images(), arch) do + case Map.fetch(Hyper.Cfg.VmLinux.images(), arch) do {:ok, path} -> path @@ -44,7 +44,7 @@ defmodule Hyper.Node.Vmlinux do @spec test_system() :: :ok | {:error, term()} def test_system do with {:ok, arch} <- Sys.Arch.current() do - case Map.fetch(Hyper.Cfg.Vmlinux.images(), arch) do + case Map.fetch(Hyper.Cfg.VmLinux.images(), arch) do {:ok, path} -> present(path) diff --git a/test/hyper/cfg/vm_linux_test.exs b/test/hyper/cfg/vm_linux_test.exs new file mode 100644 index 00000000..3190d608 --- /dev/null +++ b/test/hyper/cfg/vm_linux_test.exs @@ -0,0 +1,31 @@ +defmodule Hyper.Cfg.VmLinuxTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.VmLinux + alias Hyper.Cfg.Toml + + setup do + Application.delete_env(:hyper, VmLinux) + Toml.put_cache(%{}) + on_exit(fn -> + Application.delete_env(:hyper, VmLinux) + Toml.reload() + end) + :ok + end + + test "empty by default" do + assert VmLinux.images() == %{} + end + + test "maps amd64/aarch64 keys to arch atoms from config.exs" do + Application.put_env(:hyper, VmLinux, amd64: "/k/amd64", aarch64: "/k/arm64") + assert VmLinux.images() == %{x86_64: "/k/amd64", aarch64: "/k/arm64"} + end + + test "reads from [vmlinux] toml; config.exs wins per key" do + Toml.put_cache(%{"vmlinux" => %{"amd64" => "/toml/amd64", "aarch64" => "/toml/arm64"}}) + Application.put_env(:hyper, VmLinux, amd64: "/exs/amd64") + assert VmLinux.images() == %{x86_64: "/exs/amd64", aarch64: "/toml/arm64"} + end +end diff --git a/test/hyper/cfg/vmlinux_test.exs b/test/hyper/cfg/vmlinux_test.exs deleted file mode 100644 index 4dc0dac6..00000000 --- a/test/hyper/cfg/vmlinux_test.exs +++ /dev/null @@ -1,13 +0,0 @@ -defmodule Hyper.Cfg.VmlinuxTest do - use ExUnit.Case, async: false - - test "images/0 defaults to an empty map and reads config :hyper, :vmlinux" do - Application.delete_env(:hyper, :vmlinux) - assert Hyper.Cfg.Vmlinux.images() == %{} - - Application.put_env(:hyper, :vmlinux, %{x86_64: "/k/vmlinux"}) - assert Hyper.Cfg.Vmlinux.images() == %{x86_64: "/k/vmlinux"} - after - Application.delete_env(:hyper, :vmlinux) - end -end From 198e866353fedfc391a91ac7b0caff5f7186cfaf Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 22:37:09 +0000 Subject: [PATCH 25/47] feat(cfg): Img.store dual-source; Dirs.layer_dir delegates to it --- lib/hyper/cfg/dirs.ex | 4 ++-- lib/hyper/cfg/img.ex | 16 ++++++++++++++++ test/hyper/cfg/img_test.exs | 29 +++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 test/hyper/cfg/img_test.exs diff --git a/lib/hyper/cfg/dirs.ex b/lib/hyper/cfg/dirs.ex index 4010e50d..bb3190c0 100644 --- a/lib/hyper/cfg/dirs.ex +++ b/lib/hyper/cfg/dirs.ex @@ -13,9 +13,9 @@ defmodule Hyper.Cfg.Dirs do @spec work_dir :: Path.t() def work_dir, do: get_cfg(toml: "work_dir", default: "/srv/hyper") - @doc "Read-only image layer store (`/layers`)." + @doc "Read-only image layer store. Delegates to `Hyper.Cfg.Img.store/0`." @spec layer_dir :: Path.t() - def layer_dir, do: Path.join(work_dir(), "layers") + def layer_dir, do: Hyper.Cfg.Img.store() @doc "Per-VM control/gRPC sockets (`/socks`)." @spec socket_dir :: Path.t() diff --git a/lib/hyper/cfg/img.ex b/lib/hyper/cfg/img.ex index 7d554a8c..f1e93b54 100644 --- a/lib/hyper/cfg/img.ex +++ b/lib/hyper/cfg/img.ex @@ -7,8 +7,11 @@ defmodule Hyper.Cfg.Img do * `thin_block_sectors` - dm-thin pool allocation block size. * `thin_pool_data_size` / `thin_pool_meta_size` - sparse sizes of the node's dm-thin pool backing devices. + * `store` - absolute path to the read-only layer store. """ + import Hyper.Cfg, only: [get_cfg: 1] + @chunk_sectors Application.compile_env(:hyper, :chunk_sectors, 8) @thin_block_sectors Application.compile_env(:hyper, :thin_block_sectors, 128) @@ -30,4 +33,17 @@ defmodule Hyper.Cfg.Img do @doc "Sparse size of the node's dm-thin pool metadata device." @spec thin_pool_meta_size :: Unit.Information.t() def thin_pool_meta_size, do: Unit.Information.gib(1) + + @doc """ + Absolute path to the read-only layer store. config.exs (`store:`) > `[img] store` + toml > `/layers`. + """ + @spec store :: Path.t() + def store, + do: + get_cfg( + runtime: {__MODULE__, :store}, + toml: "img.store", + default: Path.join(Hyper.Cfg.Dirs.work_dir(), "layers") + ) end diff --git a/test/hyper/cfg/img_test.exs b/test/hyper/cfg/img_test.exs new file mode 100644 index 00000000..20794902 --- /dev/null +++ b/test/hyper/cfg/img_test.exs @@ -0,0 +1,29 @@ +defmodule Hyper.Cfg.ImgTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.Img + alias Hyper.Cfg.Dirs + alias Hyper.Cfg.Toml + + setup do + Application.delete_env(:hyper, Img) + Toml.put_cache(%{}) + on_exit(fn -> + Application.delete_env(:hyper, Img) + Toml.reload() + end) + :ok + end + + test "store defaults to /layers and Dirs.layer_dir delegates" do + assert Img.store() == Path.join(Dirs.work_dir(), "layers") + assert Dirs.layer_dir() == Img.store() + end + + test "store reads [img] store from toml and config.exs wins" do + Toml.put_cache(%{"img" => %{"store" => "/mnt/layers"}}) + assert Img.store() == "/mnt/layers" + Application.put_env(:hyper, Img, store: "/exs/layers") + assert Img.store() == "/exs/layers" + end +end From d765a0d24e47d923ef9e1abb3375404127f5a2c9 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 22:43:24 +0000 Subject: [PATCH 26/47] feat(cfg): Otel config + opentelemetry exporter wiring; drop Telemetry facade --- config/runtime.exs | 48 +++++++--------------- lib/hyper/cfg/otel.ex | 71 +++++++++++++++++++++++++++++++++ lib/hyper/cfg/telemetry.ex | 9 ----- test/hyper/cfg/facades_test.exs | 26 ------------ test/hyper/cfg/otel_test.exs | 37 +++++++++++++++++ 5 files changed, 123 insertions(+), 68 deletions(-) create mode 100644 lib/hyper/cfg/otel.ex delete mode 100644 lib/hyper/cfg/telemetry.ex create mode 100644 test/hyper/cfg/otel_test.exs diff --git a/config/runtime.exs b/config/runtime.exs index b6358ecd..a0d1f7de 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -11,45 +11,27 @@ config :hyper, Hyper.Cfg.Budget, net_bw_cap: Unit.Bandwidth.gibps(1), net_bw_max_load: 0.8 -# Where to send traces. Defaults to Honeycomb; override OTEL_EXPORTER_OTLP_* -# to point at any OTLP/HTTP backend (Collector, Grafana, etc). -if config_env() != :test do - custom_endpoint = System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT") - api_key = System.get_env("HONEYCOMB_API_KEY") - - cond do - api_key not in [nil, ""] -> - config :opentelemetry_exporter, - otlp_protocol: :http_protobuf, - otlp_endpoint: custom_endpoint || "https://api.honeycomb.io", - otlp_headers: [{"x-honeycomb-team", api_key}] - - custom_endpoint not in [nil, ""] -> - # A custom OTLP backend (e.g. a local Collector) needs no Honeycomb key. - config :opentelemetry_exporter, - otlp_protocol: :http_protobuf, - otlp_endpoint: custom_endpoint, - otlp_headers: [] - - true -> - # No backend configured: exporting to the Honeycomb default with no key - # 401s on every batch. Stay silent instead (typical for local dev). Set - # HONEYCOMB_API_KEY or OTEL_EXPORTER_OTLP_ENDPOINT to enable tracing. - config :opentelemetry, traces_exporter: :none - end -end - # Operator overrides from a well-known location. An optional Elixir config file # at /etc/hyper/config.exs (override the path with HYPER_CONFIG) is merged in # last, so its values win over every default set above. An absent file is a -# no-op -- the normal case in dev and CI. Skipped under :test so the suite never +# no-op — the normal case in dev and CI. Skipped under :test so the suite never # reads host state. +# +# OpenTelemetry exporter wiring is resolved through Hyper.Cfg.Otel so that the +# operator's `config :hyper, Hyper.Cfg.Otel, ...` stanza (if present) takes +# precedence over the TOML table and environment variables. if config_env() != :test do hyper_config = System.get_env("HYPER_CONFIG") || "/etc/hyper/config.exs" - if File.exists?(hyper_config) do - for {app, kw} <- Config.Reader.read!(hyper_config, env: config_env()) do - config app, kw - end + operator = + if File.exists?(hyper_config), do: Config.Reader.read!(hyper_config, env: config_env()), else: [] + + otel_exs = get_in(operator, [:hyper, Hyper.Cfg.Otel]) || [] + + case Hyper.Cfg.Otel.exporter_options(otel_exs) do + {:ok, opts} -> config :opentelemetry_exporter, opts + :none -> config :opentelemetry, traces_exporter: :none end + + for {app, kw} <- operator, do: config(app, kw) end diff --git a/lib/hyper/cfg/otel.ex b/lib/hyper/cfg/otel.ex new file mode 100644 index 00000000..c9e8c99c --- /dev/null +++ b/lib/hyper/cfg/otel.ex @@ -0,0 +1,71 @@ +defmodule Hyper.Cfg.Otel do + @moduledoc """ + OpenTelemetry exporter configuration. Resolved from `config :hyper, + Hyper.Cfg.Otel, proto:/endpoint:/headers:` (config.exs), the `[otel]` toml + table, then the `HONEYCOMB_API_KEY` / `OTEL_EXPORTER_OTLP_ENDPOINT` env vars. + `config/runtime.exs` calls `exporter_options/1` and feeds the result to + `config :opentelemetry_exporter`. + """ + + import Hyper.Cfg, only: [fetch_cfg: 1] + + @honeycomb "https://api.honeycomb.io" + + @doc "Resolve the `:opentelemetry_exporter` options, or `:none`." + @spec exporter_options(keyword()) :: {:ok, keyword()} | :none + def exporter_options(exs) when is_list(exs) do + endpoint = pick(exs, :endpoint, "otel.endpoint") || env_endpoint() + + case endpoint do + nil -> + :none + + ep -> + {:ok, + [ + otlp_protocol: proto(pick(exs, :proto, "otel.proto")), + otlp_endpoint: ep, + otlp_headers: headers(pick(exs, :headers, "otel.headers") || env_headers()) + ]} + end + end + + @spec pick(keyword(), atom(), String.t()) :: term() | nil + defp pick(exs, key, toml) do + case Keyword.fetch(exs, key) do + {:ok, v} -> v + :error -> case fetch_cfg(toml: toml), do: ({:ok, v} -> v; :error -> nil) + end + end + + @spec proto(term()) :: :http_protobuf | :grpc + defp proto(p) when p in [:http_protobuf, :grpc], do: p + defp proto("grpc"), do: :grpc + defp proto(_), do: :http_protobuf + + @spec headers(term()) :: [{String.t(), String.t()}] + defp headers(h) when is_map(h), do: Enum.map(h, fn {k, v} -> {to_string(k), to_string(v)} end) + defp headers(h) when is_list(h), do: Enum.map(h, fn {k, v} -> {to_string(k), to_string(v)} end) + defp headers(_), do: [] + + @spec env_endpoint() :: String.t() | nil + defp env_endpoint do + cond do + nonempty(System.get_env("HONEYCOMB_API_KEY")) -> @honeycomb + ep = nonempty(System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT")) -> ep + true -> nil + end + end + + @spec env_headers() :: [{String.t(), String.t()}] + defp env_headers do + case nonempty(System.get_env("HONEYCOMB_API_KEY")) do + nil -> [] + key -> [{"x-honeycomb-team", key}] + end + end + + @spec nonempty(String.t() | nil) :: String.t() | nil + defp nonempty(s) when is_binary(s) and s != "", do: s + defp nonempty(_), do: nil +end diff --git a/lib/hyper/cfg/telemetry.ex b/lib/hyper/cfg/telemetry.ex deleted file mode 100644 index 4c2f9a3a..00000000 --- a/lib/hyper/cfg/telemetry.ex +++ /dev/null @@ -1,9 +0,0 @@ -defmodule Hyper.Cfg.Telemetry do - @moduledoc "Read-only view of the OpenTelemetry exporter config." - - @spec exporter :: atom() - def exporter, do: Application.get_env(:opentelemetry, :traces_exporter, :none) - - @spec otlp_endpoint :: String.t() | nil - def otlp_endpoint, do: Application.get_env(:opentelemetry_exporter, :otlp_endpoint) -end diff --git a/test/hyper/cfg/facades_test.exs b/test/hyper/cfg/facades_test.exs index 8d4960e7..b521be7b 100644 --- a/test/hyper/cfg/facades_test.exs +++ b/test/hyper/cfg/facades_test.exs @@ -8,30 +8,4 @@ defmodule Hyper.Cfg.FacadesTest do test "Db.repo_opts reads the Ecto repo config" do assert Keyword.keyword?(Hyper.Cfg.Db.repo_opts()) end - - test "Telemetry.exporter reflects the configured traces_exporter" do - prior = Application.get_env(:opentelemetry, :traces_exporter) - - on_exit(fn -> - if prior == nil, - do: Application.delete_env(:opentelemetry, :traces_exporter), - else: Application.put_env(:opentelemetry, :traces_exporter, prior) - end) - - Application.put_env(:opentelemetry, :traces_exporter, :none) - assert Hyper.Cfg.Telemetry.exporter() == :none - end - - test "Telemetry.otlp_endpoint reads the configured endpoint" do - prior = Application.get_env(:opentelemetry_exporter, :otlp_endpoint) - - on_exit(fn -> - if prior == nil, - do: Application.delete_env(:opentelemetry_exporter, :otlp_endpoint), - else: Application.put_env(:opentelemetry_exporter, :otlp_endpoint, prior) - end) - - Application.put_env(:opentelemetry_exporter, :otlp_endpoint, "http://x:4318") - assert Hyper.Cfg.Telemetry.otlp_endpoint() == "http://x:4318" - end end diff --git a/test/hyper/cfg/otel_test.exs b/test/hyper/cfg/otel_test.exs new file mode 100644 index 00000000..73af045c --- /dev/null +++ b/test/hyper/cfg/otel_test.exs @@ -0,0 +1,37 @@ +defmodule Hyper.Cfg.OtelTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.Otel + alias Hyper.Cfg.Toml + + setup do + Toml.put_cache(%{}) + on_exit(fn -> Toml.reload() end) + :ok + end + + test "config.exs proto/endpoint/headers produce opentelemetry_exporter opts" do + {:ok, opts} = + Otel.exporter_options( + proto: :http_protobuf, + endpoint: "https://api.honeycomb.io", + headers: %{"x-honeycomb-team" => "KEY"} + ) + + assert opts[:otlp_protocol] == :http_protobuf + assert opts[:otlp_endpoint] == "https://api.honeycomb.io" + assert opts[:otlp_headers] == [{"x-honeycomb-team", "KEY"}] + end + + test "reads [otel] toml when config.exs is empty" do + Toml.put_cache(%{"otel" => %{"proto" => "http_protobuf", "endpoint" => "http://collector:4318"}}) + {:ok, opts} = Otel.exporter_options([]) + assert opts[:otlp_protocol] == :http_protobuf + assert opts[:otlp_endpoint] == "http://collector:4318" + assert opts[:otlp_headers] == [] + end + + test ":none when no endpoint anywhere" do + assert Otel.exporter_options([]) == :none + end +end From 4c1f26490be17d925cec3786371a92379b77998b Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 22:51:24 +0000 Subject: [PATCH 27/47] feat(cfg): Img.Db config + Ecto Repo init/2 merge; drop Db facade --- lib/hyper/cfg/db.ex | 6 ------ lib/hyper/cfg/img/db.ex | 17 +++++++++++++++++ lib/hyper/img/db/repo.ex | 5 +++++ test/hyper/cfg/facades_test.exs | 4 ---- test/hyper/cfg/img_db_test.exs | 23 +++++++++++++++++++++++ 5 files changed, 45 insertions(+), 10 deletions(-) delete mode 100644 lib/hyper/cfg/db.ex create mode 100644 lib/hyper/cfg/img/db.ex create mode 100644 test/hyper/cfg/img_db_test.exs diff --git a/lib/hyper/cfg/db.ex b/lib/hyper/cfg/db.ex deleted file mode 100644 index 597de8f2..00000000 --- a/lib/hyper/cfg/db.ex +++ /dev/null @@ -1,6 +0,0 @@ -defmodule Hyper.Cfg.Db do - @moduledoc "Read-only view of the image-DB Ecto repo config." - - @spec repo_opts :: keyword() - def repo_opts, do: Application.get_env(:hyper, Hyper.Img.Db.Repo, []) -end diff --git a/lib/hyper/cfg/img/db.ex b/lib/hyper/cfg/img/db.ex new file mode 100644 index 00000000..fa4274ef --- /dev/null +++ b/lib/hyper/cfg/img/db.ex @@ -0,0 +1,17 @@ +defmodule Hyper.Cfg.Img.Db do + @moduledoc """ + Image-database (Ecto/Postgres) connection settings — `database`/`username`/ + `password`/`hostname`. These are secrets, so they are read from `config.exs` + only (`config :hyper, Hyper.Cfg.Img.Db, ...`), never the shared `config.toml`. + `Hyper.Img.Db.Repo.init/2` merges these over its compile-time defaults. + """ + + @keys [:database, :username, :password, :hostname] + + @doc "Operator-set repo options (only the keys actually set), for `Repo.init/2`." + @spec repo_opts :: keyword() + def repo_opts do + env = Application.get_env(:hyper, __MODULE__, []) + Enum.filter(env, fn {k, _} -> k in @keys end) + end +end diff --git a/lib/hyper/img/db/repo.ex b/lib/hyper/img/db/repo.ex index a0e8cb81..96ed6735 100644 --- a/lib/hyper/img/db/repo.ex +++ b/lib/hyper/img/db/repo.ex @@ -16,4 +16,9 @@ defmodule Hyper.Img.Db.Repo do use Ecto.Repo, otp_app: :hyper, adapter: Ecto.Adapters.Postgres + + @impl true + def init(_context, config) do + {:ok, Keyword.merge(config, Hyper.Cfg.Img.Db.repo_opts())} + end end diff --git a/test/hyper/cfg/facades_test.exs b/test/hyper/cfg/facades_test.exs index b521be7b..df7af674 100644 --- a/test/hyper/cfg/facades_test.exs +++ b/test/hyper/cfg/facades_test.exs @@ -4,8 +4,4 @@ defmodule Hyper.Cfg.FacadesTest do test "Cluster.topologies reads :libcluster app env" do assert is_list(Hyper.Cfg.Cluster.topologies()) end - - test "Db.repo_opts reads the Ecto repo config" do - assert Keyword.keyword?(Hyper.Cfg.Db.repo_opts()) - end end diff --git a/test/hyper/cfg/img_db_test.exs b/test/hyper/cfg/img_db_test.exs new file mode 100644 index 00000000..b1c0a2d5 --- /dev/null +++ b/test/hyper/cfg/img_db_test.exs @@ -0,0 +1,23 @@ +defmodule Hyper.Cfg.Img.DbTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.Img.Db + + setup do + on_exit(fn -> Application.delete_env(:hyper, Db) end) + :ok + end + + test "repo_opts is empty when unset" do + Application.delete_env(:hyper, Db) + assert Db.repo_opts() == [] + end + + test "returns only the set keys" do + Application.put_env(:hyper, Db, database: "prod", hostname: "db.internal") + opts = Db.repo_opts() + assert opts[:database] == "prod" + assert opts[:hostname] == "db.internal" + refute Keyword.has_key?(opts, :username) + end +end From 5adb629c92874769a971f22fc69a58f654ddf818 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 22:55:54 +0000 Subject: [PATCH 28/47] refactor(cfg): reconcile config files + formatting/credo with phase-2 modules - runtime.exs: add cpu_max_cap default to Budget block - mix format the phase-2 modules (drift from subagent commits) - credo: sigil @doc in Unit.Time, alphabetize aliases in img/vm_linux tests --- config/runtime.exs | 5 ++++- lib/hyper/cfg/budget.ex | 1 + lib/hyper/cfg/gc.ex | 3 ++- lib/hyper/cfg/grpc.ex | 4 +++- lib/hyper/cfg/otel.ex | 11 +++++++++-- lib/unit/bandwidth.ex | 7 +++++-- lib/unit/information.ex | 7 +++++-- lib/unit/time.ex | 2 +- test/hyper/cfg/budget_test.exs | 2 ++ test/hyper/cfg/gc_test.exs | 7 ++++++- test/hyper/cfg/grpc_test.exs | 2 ++ test/hyper/cfg/img_test.exs | 4 +++- test/hyper/cfg/otel_test.exs | 5 ++++- test/hyper/cfg/vm_linux_test.exs | 4 +++- test/unit/bandwidth_parse_test.exs | 2 +- test/unit/information_parse_test.exs | 2 +- test/unit/time_parse_test.exs | 2 +- 17 files changed, 53 insertions(+), 17 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index a0d1f7de..6df3f4e9 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -6,6 +6,7 @@ config :hyper, Hyper.Cfg.Budget, mem_max: Unit.Information.gib(4), disk_max: Unit.Information.gib(4), cpu_max_load: 0.8, + cpu_max_cap: 4.0, disk_bw_cap: Unit.Bandwidth.gibps(1), disk_bw_max_load: 0.8, net_bw_cap: Unit.Bandwidth.gibps(1), @@ -24,7 +25,9 @@ if config_env() != :test do hyper_config = System.get_env("HYPER_CONFIG") || "/etc/hyper/config.exs" operator = - if File.exists?(hyper_config), do: Config.Reader.read!(hyper_config, env: config_env()), else: [] + if File.exists?(hyper_config), + do: Config.Reader.read!(hyper_config, env: config_env()), + else: [] otel_exs = get_in(operator, [:hyper, Hyper.Cfg.Otel]) || [] diff --git a/lib/hyper/cfg/budget.ex b/lib/hyper/cfg/budget.ex index 7ef0c911..2d01799e 100644 --- a/lib/hyper/cfg/budget.ex +++ b/lib/hyper/cfg/budget.ex @@ -100,6 +100,7 @@ defmodule Hyper.Cfg.Budget do @spec coerce(term(), (String.t() -> {:ok, struct()} | {:error, term()}), module(), atom()) :: {:ok, struct()} | {:error, term()} defp coerce(%mod{} = v, _parse, mod, _key), do: {:ok, v} + defp coerce(s, parse, _mod, key) when is_binary(s) do case parse.(s) do {:ok, v} -> {:ok, v} diff --git a/lib/hyper/cfg/gc.ex b/lib/hyper/cfg/gc.ex index f44539d3..96c17b96 100644 --- a/lib/hyper/cfg/gc.ex +++ b/lib/hyper/cfg/gc.ex @@ -33,7 +33,8 @@ defmodule Hyper.Cfg.Gc do def load do struct!(__MODULE__, [ {:enabled, get_cfg(runtime: {__MODULE__, :enabled}, toml: "img.gc.enabled", default: true)}, - {:batch_size, get_cfg(runtime: {__MODULE__, :batch_size}, toml: "img.gc.batch_size", default: 200)}, + {:batch_size, + get_cfg(runtime: {__MODULE__, :batch_size}, toml: "img.gc.batch_size", default: 200)}, {:batch_pause, duration(:batch_pause, "img.gc.batch_pause", Unit.Time.ms(100))}, {:sweep_interval, duration(:sweep_interval, "img.gc.sweep_interval", Unit.Time.s(60))}, {:acquire_interval, duration(:acquire_interval, "img.gc.acquire_interval", Unit.Time.s(5))}, diff --git a/lib/hyper/cfg/grpc.ex b/lib/hyper/cfg/grpc.ex index 68943f67..908d0894 100644 --- a/lib/hyper/cfg/grpc.ex +++ b/lib/hyper/cfg/grpc.ex @@ -45,13 +45,15 @@ defmodule Hyper.Cfg.Grpc do enabled: get_cfg(runtime: {__MODULE__, :enabled}, toml: "grpc.enabled", default: false), port: get_cfg(runtime: {__MODULE__, :port}, toml: "grpc.port", default: @default_port), cred: cred(get_cfg(runtime: {__MODULE__, :cred}, toml: "grpc.cred", default: nil)), - adapter_opts: get_cfg(runtime: {__MODULE__, :adapter_opts}, toml: "grpc.adapter_opts", default: []) + adapter_opts: + get_cfg(runtime: {__MODULE__, :adapter_opts}, toml: "grpc.adapter_opts", default: []) } end @spec cred(term()) :: GRPC.Credential.t() | nil defp cred(nil), do: nil defp cred(%GRPC.Credential{} = c), do: c + defp cred(%{"cert" => cert, "key" => key}), do: GRPC.Credential.new(ssl: [certfile: cert, keyfile: key]) diff --git a/lib/hyper/cfg/otel.ex b/lib/hyper/cfg/otel.ex index c9e8c99c..6490b300 100644 --- a/lib/hyper/cfg/otel.ex +++ b/lib/hyper/cfg/otel.ex @@ -33,8 +33,15 @@ defmodule Hyper.Cfg.Otel do @spec pick(keyword(), atom(), String.t()) :: term() | nil defp pick(exs, key, toml) do case Keyword.fetch(exs, key) do - {:ok, v} -> v - :error -> case fetch_cfg(toml: toml), do: ({:ok, v} -> v; :error -> nil) + {:ok, v} -> + v + + :error -> + case fetch_cfg(toml: toml), + do: ( + {:ok, v} -> v + :error -> nil + ) end end diff --git a/lib/unit/bandwidth.ex b/lib/unit/bandwidth.ex index 7a158952..b0f973c1 100644 --- a/lib/unit/bandwidth.ex +++ b/lib/unit/bandwidth.ex @@ -44,8 +44,11 @@ defmodule Unit.Bandwidth do @spec parse(String.t()) :: {:ok, t()} | {:error, {:bad_unit, String.t()}} def parse(s) when is_binary(s) do case Regex.run(~r/^\s*(\d+)\s*(Bps|KiBps|MiBps|GiBps|TiBps)\s*$/, s) do - [_, n, suffix] -> {:ok, %__MODULE__{bytes_per_sec: String.to_integer(n) * Map.fetch!(@units, suffix)}} - _ -> {:error, {:bad_unit, s}} + [_, n, suffix] -> + {:ok, %__MODULE__{bytes_per_sec: String.to_integer(n) * Map.fetch!(@units, suffix)}} + + _ -> + {:error, {:bad_unit, s}} end end diff --git a/lib/unit/information.ex b/lib/unit/information.ex index 4e79b1cf..2f7ead45 100644 --- a/lib/unit/information.ex +++ b/lib/unit/information.ex @@ -50,8 +50,11 @@ defmodule Unit.Information do @spec parse(String.t()) :: {:ok, t()} | {:error, {:bad_unit, String.t()}} def parse(s) when is_binary(s) do case Regex.run(~r/^\s*(\d+)\s*(B|KiB|MiB|GiB|TiB)\s*$/, s) do - [_, n, suffix] -> {:ok, %__MODULE__{bytes: String.to_integer(n) * Map.fetch!(@units, suffix)}} - _ -> {:error, {:bad_unit, s}} + [_, n, suffix] -> + {:ok, %__MODULE__{bytes: String.to_integer(n) * Map.fetch!(@units, suffix)}} + + _ -> + {:error, {:bad_unit, s}} end end diff --git a/lib/unit/time.ex b/lib/unit/time.ex index 5f7f5f01..72714441 100644 --- a/lib/unit/time.ex +++ b/lib/unit/time.ex @@ -52,7 +52,7 @@ defmodule Unit.Time do "h" => 3600 * @s } - @doc "Parse a duration string like `\"60s\"`/`\"100ms\"`/`\"1h\"`. Suffixes: ns/us/ms/s/m/h." + @doc ~s(Parse a duration string like `"60s"`/`"100ms"`/`"1h"`. Suffixes: ns/us/ms/s/m/h.) @spec parse(String.t()) :: {:ok, t()} | {:error, {:bad_unit, String.t()}} def parse(s) when is_binary(s) do case Regex.run(~r/^\s*(\d+)\s*(ns|us|ms|s|m|h)\s*$/, s) do diff --git a/test/hyper/cfg/budget_test.exs b/test/hyper/cfg/budget_test.exs index 452cc993..f2f150f4 100644 --- a/test/hyper/cfg/budget_test.exs +++ b/test/hyper/cfg/budget_test.exs @@ -7,10 +7,12 @@ defmodule Hyper.Cfg.BudgetTest do setup do Application.delete_env(:hyper, Budget) Toml.put_cache(%{}) + on_exit(fn -> Application.delete_env(:hyper, Budget) Toml.reload() end) + :ok end diff --git a/test/hyper/cfg/gc_test.exs b/test/hyper/cfg/gc_test.exs index 4dd23419..c0e7e061 100644 --- a/test/hyper/cfg/gc_test.exs +++ b/test/hyper/cfg/gc_test.exs @@ -7,10 +7,12 @@ defmodule Hyper.Cfg.GcTest do setup do Application.delete_env(:hyper, Gc) Toml.put_cache(%{}) + on_exit(fn -> Application.delete_env(:hyper, Gc) Toml.reload() end) + :ok end @@ -24,7 +26,10 @@ defmodule Hyper.Cfg.GcTest do end test "reads durations from [img.gc] toml as strings" do - Toml.put_cache(%{"img" => %{"gc" => %{"sweep_interval" => "30s", "grace_period" => "1h", "batch_size" => 50}}}) + Toml.put_cache(%{ + "img" => %{"gc" => %{"sweep_interval" => "30s", "grace_period" => "1h", "batch_size" => 50}} + }) + cfg = Gc.load() assert cfg.sweep_interval == Unit.Time.s(30) assert cfg.grace_period == Unit.Time.s(3600) diff --git a/test/hyper/cfg/grpc_test.exs b/test/hyper/cfg/grpc_test.exs index cf241a8c..c2bdb519 100644 --- a/test/hyper/cfg/grpc_test.exs +++ b/test/hyper/cfg/grpc_test.exs @@ -7,10 +7,12 @@ defmodule Hyper.Cfg.GrpcTest do setup do Application.delete_env(:hyper, Grpc) Toml.put_cache(%{}) + on_exit(fn -> Application.delete_env(:hyper, Grpc) Toml.reload() end) + :ok end diff --git a/test/hyper/cfg/img_test.exs b/test/hyper/cfg/img_test.exs index 20794902..ba278b0a 100644 --- a/test/hyper/cfg/img_test.exs +++ b/test/hyper/cfg/img_test.exs @@ -1,17 +1,19 @@ defmodule Hyper.Cfg.ImgTest do use ExUnit.Case, async: false - alias Hyper.Cfg.Img alias Hyper.Cfg.Dirs + alias Hyper.Cfg.Img alias Hyper.Cfg.Toml setup do Application.delete_env(:hyper, Img) Toml.put_cache(%{}) + on_exit(fn -> Application.delete_env(:hyper, Img) Toml.reload() end) + :ok end diff --git a/test/hyper/cfg/otel_test.exs b/test/hyper/cfg/otel_test.exs index 73af045c..700c39c3 100644 --- a/test/hyper/cfg/otel_test.exs +++ b/test/hyper/cfg/otel_test.exs @@ -24,7 +24,10 @@ defmodule Hyper.Cfg.OtelTest do end test "reads [otel] toml when config.exs is empty" do - Toml.put_cache(%{"otel" => %{"proto" => "http_protobuf", "endpoint" => "http://collector:4318"}}) + Toml.put_cache(%{ + "otel" => %{"proto" => "http_protobuf", "endpoint" => "http://collector:4318"} + }) + {:ok, opts} = Otel.exporter_options([]) assert opts[:otlp_protocol] == :http_protobuf assert opts[:otlp_endpoint] == "http://collector:4318" diff --git a/test/hyper/cfg/vm_linux_test.exs b/test/hyper/cfg/vm_linux_test.exs index 3190d608..a3c166d8 100644 --- a/test/hyper/cfg/vm_linux_test.exs +++ b/test/hyper/cfg/vm_linux_test.exs @@ -1,16 +1,18 @@ defmodule Hyper.Cfg.VmLinuxTest do use ExUnit.Case, async: false - alias Hyper.Cfg.VmLinux alias Hyper.Cfg.Toml + alias Hyper.Cfg.VmLinux setup do Application.delete_env(:hyper, VmLinux) Toml.put_cache(%{}) + on_exit(fn -> Application.delete_env(:hyper, VmLinux) Toml.reload() end) + :ok end diff --git a/test/unit/bandwidth_parse_test.exs b/test/unit/bandwidth_parse_test.exs index e9154ddd..31f1ef76 100644 --- a/test/unit/bandwidth_parse_test.exs +++ b/test/unit/bandwidth_parse_test.exs @@ -18,7 +18,7 @@ defmodule Unit.BandwidthParseTest do end property "parse! inverts gibps" do - check all n <- integer(0..1024) do + check all(n <- integer(0..1024)) do assert Bandwidth.parse!("#{n}GiBps") == Bandwidth.gibps(n) end end diff --git a/test/unit/information_parse_test.exs b/test/unit/information_parse_test.exs index ce8f7fde..6a971b19 100644 --- a/test/unit/information_parse_test.exs +++ b/test/unit/information_parse_test.exs @@ -22,7 +22,7 @@ defmodule Unit.InformationParseTest do end property "parse! inverts the gib constructor" do - check all n <- integer(0..4096) do + check all(n <- integer(0..4096)) do assert Information.parse!("#{n}GiB") == Information.gib(n) end end diff --git a/test/unit/time_parse_test.exs b/test/unit/time_parse_test.exs index 4c5c35e1..df74dace 100644 --- a/test/unit/time_parse_test.exs +++ b/test/unit/time_parse_test.exs @@ -21,7 +21,7 @@ defmodule Unit.TimeParseTest do end property "parse! inverts s" do - check all n <- integer(0..100_000) do + check all(n <- integer(0..100_000)) do assert Time.parse!("#{n}s") == Time.s(n) end end From ec6e1c1ab2486f541a405b70c5f432bdcafeac29 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Fri, 26 Jun 2026 23:02:05 +0000 Subject: [PATCH 29/47] docs(cfg): note [grpc] toml source + cred coercion; hoist import to top --- lib/hyper/cfg/grpc.ex | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/hyper/cfg/grpc.ex b/lib/hyper/cfg/grpc.ex index 908d0894..1d2adbec 100644 --- a/lib/hyper/cfg/grpc.ex +++ b/lib/hyper/cfg/grpc.ex @@ -19,12 +19,19 @@ defmodule Hyper.Cfg.Grpc do Build the credential where you load your keys (e.g. `config/runtime.exs`); Hyper never reads the filesystem on your behalf. + Each field also reads from the `[grpc]` table of `config.toml` when not set in + `config.exs`. There, `cred` is given as an inline table + `{ cert = "/path/cert.pem", key = "/path/key.pem" }`, which is coerced into a + `GRPC.Credential`. + > #### Co-located nodes {: .info} > > Every node binds `:port`. Running multiple nodes on one host (e.g. a local > cluster) requires giving each a distinct port via its own config. """ + import Hyper.Cfg, only: [get_cfg: 1] + @default_port 50_051 defstruct enabled: false, port: @default_port, cred: nil, adapter_opts: [] @@ -36,8 +43,6 @@ defmodule Hyper.Cfg.Grpc do adapter_opts: keyword() } - import Hyper.Cfg, only: [get_cfg: 1] - @doc "Load the gRPC server configuration: config.exs > [grpc] toml > defaults." @spec load() :: t() def load do From 06f7a273f30bc7400bc5a2f48c00a53c9abf3522 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 00:16:34 +0000 Subject: [PATCH 30/47] docs(cookbook): reconcile config.md with phase-2 dual-source config --- docs/cookbook/config.md | 432 ++++++++++++++++++++++++++++++++++------ 1 file changed, 367 insertions(+), 65 deletions(-) diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md index ab4d8f59..2d156220 100644 --- a/docs/cookbook/config.md +++ b/docs/cookbook/config.md @@ -4,12 +4,12 @@ Configuring `Hyper` is done through four layers, in priority: ## Configuration Files -| File | Description | -|------|-------------| -| `/etc/hyper/config.exs` | The `config.exs` file is exlusively used by the unprivileged `hyper` application. The purpose of this file is to allow you to load configuration values at runtime. If you are using a secrets manager, this is the right place to load the secrets. Must be owned by `root` and only writeable by `root`. | +| File | Description | +| ------------------------ | ----------- | +| `/etc/hyper/config.exs` | The `config.exs` file is exlusively used by the unprivileged `hyper` application. The purpose of this file is to allow you to load configuration values at runtime. If you are using a secrets manager, this is the right place to load the secrets. Must be owned by `root` and only writeable by `root`. | | `/etc/hyper/config.toml` | The `/etc/hyper/config.toml` file is used for static configuration. Unlike `config.exs`, it is used by both `Hyper` and `hyper-suidhelper` which means that it can impact the behavior of a process running under `root`. Must be owned by `root` and only writable by `root`. | | Compile-Time `config.ex` | The compile-time configuration is generally used to fine-tune the performance of Hyper. You likely do not need to edit most of the configuration fields exposed by this file for day-to-day usage, but they are available for you to tweak. | -| Defaults | `Hyper` has a set of sane defaults for some, but not all config fields. | +| Defaults | `Hyper` has a set of sane defaults for some, but not all config fields. | **Note that not all layers allow all configuration fields to be tweaked.** Read further for more details on where and how each configuration field is set. @@ -26,46 +26,106 @@ Note the keys are abbreviated for better layout: configuration section expands to `:hyper, Hyper.Cfg.Tools, mke2fs: "/path/to/mke2fs"`. -## Root Keys (`Hyper.Config`, `-`) +## Root Keys (`Hyper.Cfg`, `-`) -| Config Key | `config.exs` | `config.toml` | Default | Notes | -|---------------|-------------------------|--------------------------|-----------------------------------|-------------------------------------------------------------------------| -| `work_dir` | - | `work_dir` | - | [Absolute Path](#absolute-path) where `Hyper` creates its working tree. | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | ------- | ----- | +| `work_dir` | - | `work_dir` | - | [Absolute Path](#absolute-path) where `Hyper` creates its working tree. | + + +### `config.exs` + +```elixir +# `work_dir` is shared with hyper-suidhelper, so it is set in config.toml only. +``` + +### `config.toml` + +```toml +work_dir = "/srv/hyper" +``` + ## Tool Configuration (`Hyper.Cfg.Tools`, `[tools]`) Hyper relies on a large number of external tools, of which the paths are configurable: -| Config Key | `config.exs` | `config.toml` | Default | Notes | -|---------------|-------------------------|--------------------------|-------------------------------------|---------------------------------| -| `firecracker` | - | `.firecracker` | - | [Safe Path](#safe-path) | -| `jailer` | - | `.jailer` | - | [Safe Path](#safe-path) | -| `dmsetup` | - | `.dmsetup` | `"/usr/sbin/dmsetup"` | [Safe Path](#safe-path) | -| `losetup` | - | `.losetup` | `"/usr/sbin/losetup"` | [Safe Path](#safe-path) | -| `blockdev` | - | `.blockdev` | `"/usr/sbin/blockdev"` | [Safe Path](#safe-path) | -| `mke2fs` | `.mke2fs` | `.losetup` | `$PATH["mke2fs"]` | | -| `skopeo` | `.skopeo` | `.skopeo` | `$PATH["skopeo"]` | | -| `umoci` | `.umoci` | `.umoci` | Automatically downloaded. | | -| `suidhelper` | `.suidhelper` | `.suidhelper` | `"/usr/local/bin/hyper-suidhelper"` | [Absolute Path](#absolute-path) | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ------------- | ------------- | -------------- | ----------------------------------- | ----- | +| `firecracker` | - | `.firecracker` | - | [Safe Path](#safe-path) | +| `jailer` | - | `.jailer` | - | [Safe Path](#safe-path) | +| `dmsetup` | - | `.dmsetup` | `"/usr/sbin/dmsetup"` | [Safe Path](#safe-path) | +| `losetup` | - | `.losetup` | `"/usr/sbin/losetup"` | [Safe Path](#safe-path) | +| `blockdev` | - | `.blockdev` | `"/usr/sbin/blockdev"` | [Safe Path](#safe-path) | +| `mke2fs` | `.mke2fs` | `.mke2fs` | `$PATH["mke2fs"]` | | +| `skopeo` | `.skopeo` | `.skopeo` | `$PATH["skopeo"]` | | +| `umoci` | `.umoci` | `.umoci` | Automatically downloaded. | | +| `suidhelper` | `.suidhelper` | `.suidhelper` | `"/usr/local/bin/hyper-suidhelper"` | [Absolute Path](#absolute-path) | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Tools, + skopeo: "/usr/local/bin/skopeo", + mke2fs: "/usr/local/bin/mke2fs", + umoci: "/usr/local/bin/umoci", + suidhelper: "/usr/local/bin/hyper-suidhelper" +``` + +### `config.toml` + +```toml +[tools] +# Privileged tools -- config.toml only (shared with the suidhelper). +firecracker = "/opt/firecracker/firecracker" +jailer = "/opt/firecracker/jailer" +dmsetup = "/usr/sbin/dmsetup" +losetup = "/usr/sbin/losetup" +blockdev = "/usr/sbin/blockdev" + +# Node tools -- may also be set in config.exs, which wins when both are set. +skopeo = "/usr/local/bin/skopeo" +mke2fs = "/usr/local/bin/mke2fs" +umoci = "/usr/local/bin/umoci" +suidhelper = "/usr/local/bin/hyper-suidhelper" +``` + ## Jail Confinement (`-`, `[jails]`) -| Config Key | `config.exs` | `config.toml` | Default | Notes | -|----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| `cgroup` | - | `.cgroup` | `"hyper"` | Parent cgroup under which each VM's cgroup is nested. Each VM receives its own ephemeral cgroup which lives under the umbrella of this cgroup. | -| `uid_gid_range`| - | `.uid_gid_range` | - | [Range](#range) limiting the UID/GID values given to VMs. Each VM receives its own UID/GID pair, within these bounds. Must not be an existing user/group. | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| --------------- | ------------ | ---------------- | --------- | ----- | +| `cgroup` | - | `.cgroup` | `"hyper"` | Parent cgroup under which each VM's cgroup is nested. Each VM receives its own ephemeral cgroup which lives under the umbrella of this cgroup. | +| `uid_gid_range` | - | `.uid_gid_range` | - | [Range](#range) limiting the UID/GID values given to VMs. Each VM receives its own UID/GID pair, within these bounds. Must not be an existing user/group. | + + +### `config.exs` + +```elixir +# Jail confinement is shared with hyper-suidhelper, so it is set in config.toml only. +``` + +### `config.toml` + +```toml +[jails] +cgroup = "hyper" +uid_gid_range = [900000, 999999] +``` + ## gRPC Configuration (`Hyper.Cfg.Grpc`, `[grpc]`) Hyper supports a [gRPC](https://grpc.io/) interface enabling you to interface with `Hyper` from any language. -| Config Key | `config.exs` | `config.toml`| Default | Notes | -|------------|-------------------------------------|-------------------------|---------|--------------------------------------------------------------------------------------------------------| -| `enabled` | `.enabled` | `.enabled` | `false` | | -| `port` | `.port` | `.port` | `50051` | The port on which to serve the interface. | -| `cred` | `.cred` | `.cred` | `nil` | Either a `GRPC.Credential` or a TOML struct `{ cert = "/path/to/cert.pem", key = "/path/to/key.pem"}`. Cleartext mode when `nil`. | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | ------- | ----- | +| `enabled` | `.enabled` | `.enabled` | `false` | | +| `port` | `.port` | `.port` | `50051` | The port on which to serve the interface. | +| `cred` | `.cred` | `.cred` | `nil` | Either a `GRPC.Credential` or a TOML struct `{ cert = "/path/to/cert.pem", key = "/path/to/key.pem"}`. Cleartext mode when `nil`. | > #### Uniqueness {: .info} > @@ -77,61 +137,186 @@ with `Hyper` from any language. > you can also conditionally enable the `gRPC` server based on logic in your > `config.exs`, for example, to only spawn it on your "main" server. + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Grpc, + enabled: true, + port: 50_051, + cred: + GRPC.Credential.new( + ssl: [certfile: "/etc/hyper/tls/cert.pem", keyfile: "/etc/hyper/tls/key.pem"] + ) +``` + +### `config.toml` + +```toml +[grpc] +enabled = true +port = 50051 +cred = { cert = "/etc/hyper/tls/cert.pem", key = "/etc/hyper/tls/key.pem" } +``` + + ## Telemetry Configuration (`Hyper.Cfg.Otel`, `[otel]`) You can configure telemetry with Hyper by adding this section to your configuration and Hyper will emit tracing spans as configured. -| Config Key | `config.exs` | `config.toml`| Default | Notes | -|------------|-------------------------------------|-------------------------|---------|--------------------------------------------------------------------------------------------------------| -| `proto` | `.proto` | `.proto` | - | | -| `endpoint` | `.endpoint` | `.endpoint` | - | | -| `headers` | `.headers` | `.headers` | - | | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | ------- | ----- | +| `proto` | `.proto` | `.proto` | - | | +| `endpoint` | `.endpoint` | `.endpoint` | - | | +| `headers` | `.headers` | `.headers` | - | | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Otel, + proto: :http_protobuf, + endpoint: "https://api.honeycomb.io", + headers: %{"x-honeycomb-team" => "YOUR_API_KEY"} +``` + +### `config.toml` + +```toml +[otel] +proto = "http_protobuf" +endpoint = "https://api.honeycomb.io" +headers = { "x-honeycomb-team" = "YOUR_API_KEY" } +``` + ## Budget Configuration (`Hyper.Cfg.Budget`, `[budget]`) Hyper allows you to control the absolute maximal budgets that are available to all VMs on a particular node. -| Config Key | `config.exs` | `config.toml` | Default | Notes | -|--------------------|---------------------------------------|---------------------------|-----------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| `mem_max` | `.mem_max` | `.mem_max` | `"hyper"` | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of RAM usage. This value **must not** exceed available system memory. | -| `disk_max` | `.disk_max` | `.disk_max` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of disk usage. This value **must not** exceed available system disk space. | -| `cpu_max_load` | `.cpu_max_load` | `.cpu_max_load` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of CPU usage. | -| `cpu_max_cap` | `.cpu_max_cap` | `.cpu_max_cap` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of CPU usage. | -| `disk_bw_cap` | `.disk_bw_cap` | `.disk_bw_cap` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of disk bandwidth. | -| `disk_bw_max_load` | `.disk_bw_max_load` | `.disk_bw_max_load` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of disk bandwidth. | -| `net_bw_cap` | `.net_bw_cap` | `.net_bw_cap` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of net bandwidth. | -| `net_bw_max_load` | `.net_bw_max_load` | `.net_bw_max_load` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of net bandwidth. | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ------------------ | ------------------- | ------------------- | --------- | ----- | +| `mem_max` | `.mem_max` | `.mem_max` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of RAM usage. This value **must not** exceed available system memory. | +| `disk_max` | `.disk_max` | `.disk_max` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of disk usage. This value **must not** exceed available system disk space. | +| `cpu_max_load` | `.cpu_max_load` | `.cpu_max_load` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of CPU usage. | +| `cpu_max_cap` | `.cpu_max_cap` | `.cpu_max_cap` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of CPU usage. | +| `disk_bw_cap` | `.disk_bw_cap` | `.disk_bw_cap` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of disk bandwidth. | +| `disk_bw_max_load` | `.disk_bw_max_load` | `.disk_bw_max_load` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of disk bandwidth. | +| `net_bw_cap` | `.net_bw_cap` | `.net_bw_cap` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of net bandwidth. | +| `net_bw_max_load` | `.net_bw_max_load` | `.net_bw_max_load` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of net bandwidth. | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Budget, + mem_max: Unit.Information.gib(4), + disk_max: Unit.Information.gib(64), + cpu_max_load: 0.8, + cpu_max_cap: 4.0, + disk_bw_cap: Unit.Bandwidth.gibps(1), + disk_bw_max_load: 0.8, + net_bw_cap: Unit.Bandwidth.gibps(1), + net_bw_max_load: 0.8 +``` + +### `config.toml` + +```toml +[budget] +mem_max = "4GiB" +disk_max = "64GiB" +cpu_max_load = 0.8 +cpu_max_cap = 4.0 +disk_bw_cap = "1GiBps" +disk_bw_max_load = 0.8 +net_bw_cap = "1GiBps" +net_bw_max_load = 0.8 +``` + ## VmLinux Paths (`Hyper.Cfg.VmLinux`, `[vmlinux]`) Hyper requires Linux images for the architectures it runs on: -| Config Key | `config.exs` | `config.toml` | Default | Notes | -|----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| `amd64` | `.amd64` | `.amd64` | Automatically downloaded from [hyper-vmlinux](https://github.com/harmont-dev/hyper-vmlinux). | [Absolute Path](#absolute-path). | -| `aarch64`| `.aarch64` | `.aarch64` | Automatically downloaded from [hyper-vmlinux](https://github.com/harmont-dev/hyper-vmlinux). | [Absolute Path](#absolute-path). | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | -------------------------------------------------------------------------------------------- | ----- | +| `amd64` | `.amd64` | `.amd64` | Automatically downloaded from [hyper-vmlinux](https://github.com/harmont-dev/hyper-vmlinux). | [Absolute Path](#absolute-path). | +| `aarch64` | `.aarch64` | `.aarch64` | Automatically downloaded from [hyper-vmlinux](https://github.com/harmont-dev/hyper-vmlinux). | [Absolute Path](#absolute-path). | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.VmLinux, + amd64: "/srv/hyper/redist/vmlinux/vmlinux-amd64", + aarch64: "/srv/hyper/redist/vmlinux/vmlinux-aarch64" +``` + +### `config.toml` + +```toml +[vmlinux] +amd64 = "/srv/hyper/redist/vmlinux/vmlinux-amd64" +aarch64 = "/srv/hyper/redist/vmlinux/vmlinux-aarch64" +``` + ## Image Configuration (`Hyper.Cfg.Img`, `[img]`) Hyper's image provisioning layer has a large set of configuration flags enabling you to tweak how you want Hyper to manage images. -| Config Key | `config.exs` | `config.toml` | Default | Notes | -|----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| `store` | `.store` | `.store` | - | [Absolute Path](#absolute-path) to the [layer storage medium](./architecture.md#storage). | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | ------- | ----- | +| `store` | `.store` | `.store` | - | [Absolute Path](#absolute-path) to the [layer storage medium](./architecture.md#storage). | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Img, + store: "/srv/hyper/layers" +``` + +### `config.toml` + +```toml +[img] +store = "/srv/hyper/layers" +``` + Additionally, sub-sections are available. ### Database Configuration (`Hyper.Cfg.Img.Db`, `[img.db]`) -| Config Key | `config.exs` | `config.toml` | Default | Notes | -|----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| `database` | `.database` | - | `"hyper"` | | -| `username` | `.username` | - | - | | -| `password` | `.password` | - | - | | -| `hostname` | `.hostname` | - | - | | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | --------- | ----- | +| `database` | `.database` | - | `"hyper"` | | +| `username` | `.username` | - | - | | +| `password` | `.password` | - | - | | +| `hostname` | `.hostname` | - | - | | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Img.Db, + database: "hyper", + username: "hyper", + password: System.fetch_env!("HYPER_DB_PASSWORD"), + hostname: "db.internal" +``` + +### `config.toml` + +```toml +# Database credentials are secrets -- set them in config.exs only. +``` + ### Garbage Collector Configuration (`Hyper.Cfg.Img.Gc`, `[img.gc]`) @@ -142,15 +327,43 @@ unusable. This is always enabled. Since this scans through the whole layer database, it can have an impact on performance, and tweaking it may be necessary. -| Config Key | `config.exs` | `config.toml` | Default | Notes | -|----------------|-------------------------|--------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| `batch_size` | `.batch_size` | `.batch_size` | `"hyper"` | `200` | -| `batch_pause` | `.batch_pause` | `.batch_pause` | `100ms` | | -| `sweep_interval` | `.sweep_interval` | `.sweep_interval` | `60s` | | -| `acquire_interval` | `.acquire_interval` | `.acquire_interval` | `5s` | | -| `retry` | `.retry` | `.retry` | `60s` | | -| `timeout` | `.timeout` | `.timeout` | `5s` | | -| `grace_period` | `.grace_period` | `.grace_period` | `1h` | | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ------------------ | ------------------- | ------------------- | --------- | ----- | +| `batch_size` | `.batch_size` | `.batch_size` | `200` | | +| `batch_pause` | `.batch_pause` | `.batch_pause` | `100ms` | | +| `sweep_interval` | `.sweep_interval` | `.sweep_interval` | `60s` | | +| `acquire_interval` | `.acquire_interval` | `.acquire_interval` | `5s` | | +| `retry` | `.retry` | `.retry` | `60s` | | +| `timeout` | `.timeout` | `.timeout` | `5s` | | +| `grace_period` | `.grace_period` | `.grace_period` | `1h` | | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Img.Gc, + batch_size: 200, + batch_pause: Unit.Time.ms(100), + sweep_interval: Unit.Time.s(60), + acquire_interval: Unit.Time.s(5), + retry: Unit.Time.s(60), + timeout: Unit.Time.s(5), + grace_period: Unit.Time.s(3600) +``` + +### `config.toml` + +```toml +[img.gc] +batch_size = 200 +batch_pause = "100ms" +sweep_interval = "60s" +acquire_interval = "5s" +retry = "60s" +timeout = "5s" +grace_period = "1h" +``` + ## Cluster Topology (`Hyper.Cfg.Cluster`, `[cluster]`) @@ -181,3 +394,92 @@ necessary. #### Unit - Represents a unit as defined in `Unit.*`. + +## Total Examples + +A complete `config.exs` and `config.toml` exercising every section: + + +### `config.exs` + +```elixir +import Config + +# Node-only tools (config.exs wins over config.toml for these). +config :hyper, Hyper.Cfg.Tools, + skopeo: "/usr/local/bin/skopeo", + mke2fs: "/usr/local/bin/mke2fs" + +config :hyper, Hyper.Cfg.Grpc, + enabled: true, + port: 50_051 + +config :hyper, Hyper.Cfg.Otel, + proto: :http_protobuf, + endpoint: "https://api.honeycomb.io", + headers: %{"x-honeycomb-team" => System.fetch_env!("HONEYCOMB_API_KEY")} + +config :hyper, Hyper.Cfg.Budget, + mem_max: Unit.Information.gib(4), + disk_max: Unit.Information.gib(64), + cpu_max_load: 0.8, + cpu_max_cap: 4.0, + disk_bw_cap: Unit.Bandwidth.gibps(1), + disk_bw_max_load: 0.8, + net_bw_cap: Unit.Bandwidth.gibps(1), + net_bw_max_load: 0.8 + +config :hyper, Hyper.Cfg.Img, + store: "/srv/hyper/layers" + +config :hyper, Hyper.Cfg.Img.Db, + database: "hyper", + username: "hyper", + password: System.fetch_env!("HYPER_DB_PASSWORD"), + hostname: "db.internal" +``` + +### `config.toml` + +```toml +work_dir = "/srv/hyper" + +[tools] +firecracker = "/opt/firecracker/firecracker" +jailer = "/opt/firecracker/jailer" + +[jails] +cgroup = "hyper" +uid_gid_range = [900000, 999999] + +[grpc] +enabled = true +port = 50051 + +[otel] +proto = "http_protobuf" +endpoint = "https://api.honeycomb.io" + +[budget] +mem_max = "4GiB" +disk_max = "64GiB" +cpu_max_load = 0.8 +cpu_max_cap = 4.0 +disk_bw_cap = "1GiBps" +disk_bw_max_load = 0.8 +net_bw_cap = "1GiBps" +net_bw_max_load = 0.8 + +[vmlinux] +amd64 = "/srv/hyper/redist/vmlinux/vmlinux-amd64" +aarch64 = "/srv/hyper/redist/vmlinux/vmlinux-aarch64" + +[img] +store = "/srv/hyper/layers" + +[img.gc] +batch_size = 200 +sweep_interval = "60s" +grace_period = "1h" +``` + From 5a08065a6ed102c5305e8f754f588942ea2e9304 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 00:19:46 +0000 Subject: [PATCH 31/47] docs(cookbook): drop module/toml labels from config.md section headers --- docs/cookbook/config.md | 42 +++++++++++++---------------------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md index 2d156220..43bf9e74 100644 --- a/docs/cookbook/config.md +++ b/docs/cookbook/config.md @@ -26,19 +26,13 @@ Note the keys are abbreviated for better layout: configuration section expands to `:hyper, Hyper.Cfg.Tools, mke2fs: "/path/to/mke2fs"`. -## Root Keys (`Hyper.Cfg`, `-`) +## Root Keys | Config Key | `config.exs` | `config.toml` | Default | Notes | | ---------- | ------------ | ------------- | ------- | ----- | | `work_dir` | - | `work_dir` | - | [Absolute Path](#absolute-path) where `Hyper` creates its working tree. | -### `config.exs` - -```elixir -# `work_dir` is shared with hyper-suidhelper, so it is set in config.toml only. -``` - ### `config.toml` ```toml @@ -46,7 +40,7 @@ work_dir = "/srv/hyper" ``` -## Tool Configuration (`Hyper.Cfg.Tools`, `[tools]`) +## Tool Configuration Hyper relies on a large number of external tools, of which the paths are configurable: @@ -67,6 +61,8 @@ configurable: ### `config.exs` ```elixir +# Note that not all the tools are available here. Privileged tools, used by the +# suidhelper, cannot be configured here. config :hyper, Hyper.Cfg.Tools, skopeo: "/usr/local/bin/skopeo", mke2fs: "/usr/local/bin/mke2fs", @@ -93,7 +89,7 @@ suidhelper = "/usr/local/bin/hyper-suidhelper" ``` -## Jail Confinement (`-`, `[jails]`) +## Jail Confinement | Config Key | `config.exs` | `config.toml` | Default | Notes | | --------------- | ------------ | ---------------- | --------- | ----- | @@ -101,12 +97,6 @@ suidhelper = "/usr/local/bin/hyper-suidhelper" | `uid_gid_range` | - | `.uid_gid_range` | - | [Range](#range) limiting the UID/GID values given to VMs. Each VM receives its own UID/GID pair, within these bounds. Must not be an existing user/group. | -### `config.exs` - -```elixir -# Jail confinement is shared with hyper-suidhelper, so it is set in config.toml only. -``` - ### `config.toml` ```toml @@ -116,7 +106,7 @@ uid_gid_range = [900000, 999999] ``` -## gRPC Configuration (`Hyper.Cfg.Grpc`, `[grpc]`) +## gRPC Configuration Hyper supports a [gRPC](https://grpc.io/) interface enabling you to interface with `Hyper` from any language. @@ -160,7 +150,7 @@ cred = { cert = "/etc/hyper/tls/cert.pem", key = "/etc/hyper/tls/key.pem" } ``` -## Telemetry Configuration (`Hyper.Cfg.Otel`, `[otel]`) +## Telemetry Configuration You can configure telemetry with Hyper by adding this section to your configuration and Hyper will emit tracing spans as configured. @@ -191,7 +181,7 @@ headers = { "x-honeycomb-team" = "YOUR_API_KEY" } ``` -## Budget Configuration (`Hyper.Cfg.Budget`, `[budget]`) +## Budget Configuration Hyper allows you to control the absolute maximal budgets that are available to all VMs on a particular node. @@ -237,7 +227,7 @@ net_bw_max_load = 0.8 ``` -## VmLinux Paths (`Hyper.Cfg.VmLinux`, `[vmlinux]`) +## VmLinux Paths Hyper requires Linux images for the architectures it runs on: @@ -264,7 +254,7 @@ aarch64 = "/srv/hyper/redist/vmlinux/vmlinux-aarch64" ``` -## Image Configuration (`Hyper.Cfg.Img`, `[img]`) +## Image Configuration Hyper's image provisioning layer has a large set of configuration flags enabling you to tweak how you want Hyper to manage images. @@ -291,7 +281,7 @@ store = "/srv/hyper/layers" Additionally, sub-sections are available. -### Database Configuration (`Hyper.Cfg.Img.Db`, `[img.db]`) +### Database Configuration | Config Key | `config.exs` | `config.toml` | Default | Notes | | ---------- | ------------ | ------------- | --------- | ----- | @@ -310,15 +300,9 @@ config :hyper, Hyper.Cfg.Img.Db, password: System.fetch_env!("HYPER_DB_PASSWORD"), hostname: "db.internal" ``` - -### `config.toml` - -```toml -# Database credentials are secrets -- set them in config.exs only. -``` -### Garbage Collector Configuration (`Hyper.Cfg.Img.Gc`, `[img.gc]`) +### Garbage Collector Configuration Hyper supports a mechanism to prune unreferenced image layers. Unreferenced image layers occur when an ungraceful crash happens, resulting in entries in @@ -365,7 +349,7 @@ grace_period = "1h" ``` -## Cluster Topology (`Hyper.Cfg.Cluster`, `[cluster]`) +## Cluster Topology From 9f6ff827a3a1b36e8e47a34f09b0530e3eed95f7 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 00:20:52 +0000 Subject: [PATCH 32/47] docs(cookbook): use realistic kernel/layer-store paths in config examples --- docs/cookbook/config.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md index 43bf9e74..f0f5bc3f 100644 --- a/docs/cookbook/config.md +++ b/docs/cookbook/config.md @@ -241,16 +241,16 @@ Hyper requires Linux images for the architectures it runs on: ```elixir config :hyper, Hyper.Cfg.VmLinux, - amd64: "/srv/hyper/redist/vmlinux/vmlinux-amd64", - aarch64: "/srv/hyper/redist/vmlinux/vmlinux-aarch64" + amd64: "/opt/hyper/kernels/vmlinux-amd64", + aarch64: "/opt/hyper/kernels/vmlinux-aarch64" ``` ### `config.toml` ```toml [vmlinux] -amd64 = "/srv/hyper/redist/vmlinux/vmlinux-amd64" -aarch64 = "/srv/hyper/redist/vmlinux/vmlinux-aarch64" +amd64 = "/opt/hyper/kernels/vmlinux-amd64" +aarch64 = "/opt/hyper/kernels/vmlinux-aarch64" ``` @@ -268,14 +268,14 @@ enabling you to tweak how you want Hyper to manage images. ```elixir config :hyper, Hyper.Cfg.Img, - store: "/srv/hyper/layers" + store: "/mnt/hyper/layers" ``` ### `config.toml` ```toml [img] -store = "/srv/hyper/layers" +store = "/mnt/hyper/layers" ``` @@ -414,7 +414,7 @@ config :hyper, Hyper.Cfg.Budget, net_bw_max_load: 0.8 config :hyper, Hyper.Cfg.Img, - store: "/srv/hyper/layers" + store: "/mnt/hyper/layers" config :hyper, Hyper.Cfg.Img.Db, database: "hyper", @@ -455,11 +455,11 @@ net_bw_cap = "1GiBps" net_bw_max_load = 0.8 [vmlinux] -amd64 = "/srv/hyper/redist/vmlinux/vmlinux-amd64" -aarch64 = "/srv/hyper/redist/vmlinux/vmlinux-aarch64" +amd64 = "/opt/hyper/kernels/vmlinux-amd64" +aarch64 = "/opt/hyper/kernels/vmlinux-aarch64" [img] -store = "/srv/hyper/layers" +store = "/mnt/hyper/layers" [img.gc] batch_size = 200 From e95b7e919d93fcddd86582bc06e3842c946f38b6 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 00:28:33 +0000 Subject: [PATCH 33/47] feat(cfg): GC is non-disablable core; Mon cadence fixed (not runtime); doc grpc adapter_opts - drop Gc.enabled: the layer GC is core, always runs (Img.Db.Gc.init always starts) - Hyper.Cfg.Mon no longer reads runtime config; sampling cadence is a fixed internal - document the gRPC adapter_opts field --- docs/cookbook/config.md | 11 ++++++----- lib/hyper/cfg/gc.ex | 3 --- lib/hyper/cfg/mon.ex | 18 ++++-------------- lib/hyper/img/db/gc.ex | 8 +------- test/hyper/cfg/gc_test.exs | 1 - 5 files changed, 11 insertions(+), 30 deletions(-) diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md index f0f5bc3f..d6fa91da 100644 --- a/docs/cookbook/config.md +++ b/docs/cookbook/config.md @@ -111,11 +111,12 @@ uid_gid_range = [900000, 999999] Hyper supports a [gRPC](https://grpc.io/) interface enabling you to interface with `Hyper` from any language. -| Config Key | `config.exs` | `config.toml` | Default | Notes | -| ---------- | ------------ | ------------- | ------- | ----- | -| `enabled` | `.enabled` | `.enabled` | `false` | | -| `port` | `.port` | `.port` | `50051` | The port on which to serve the interface. | -| `cred` | `.cred` | `.cred` | `nil` | Either a `GRPC.Credential` or a TOML struct `{ cert = "/path/to/cert.pem", key = "/path/to/key.pem"}`. Cleartext mode when `nil`. | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| -------------- | --------------- | --------------- | ------- | ----- | +| `enabled` | `.enabled` | `.enabled` | `false` | | +| `port` | `.port` | `.port` | `50051` | The port on which to serve the interface. | +| `cred` | `.cred` | `.cred` | `nil` | Either a `GRPC.Credential` or a TOML struct `{ cert = "/path/to/cert.pem", key = "/path/to/key.pem"}`. Cleartext mode when `nil`. | +| `adapter_opts` | `.adapter_opts` | `.adapter_opts` | `[]` | Options forwarded to the gRPC server adapter, e.g. the bind address `ip: {0, 0, 0, 0}`. | > #### Uniqueness {: .info} > diff --git a/lib/hyper/cfg/gc.ex b/lib/hyper/cfg/gc.ex index 96c17b96..db74d372 100644 --- a/lib/hyper/cfg/gc.ex +++ b/lib/hyper/cfg/gc.ex @@ -9,7 +9,6 @@ defmodule Hyper.Cfg.Gc do import Hyper.Cfg, only: [get_cfg: 1] @type t :: %__MODULE__{ - enabled: boolean(), batch_size: pos_integer(), batch_pause: Unit.Time.t(), sweep_interval: Unit.Time.t(), @@ -19,7 +18,6 @@ defmodule Hyper.Cfg.Gc do grace_period: Unit.Time.t() } defstruct [ - :enabled, :batch_size, :batch_pause, :sweep_interval, @@ -32,7 +30,6 @@ defmodule Hyper.Cfg.Gc do @spec load :: t() def load do struct!(__MODULE__, [ - {:enabled, get_cfg(runtime: {__MODULE__, :enabled}, toml: "img.gc.enabled", default: true)}, {:batch_size, get_cfg(runtime: {__MODULE__, :batch_size}, toml: "img.gc.batch_size", default: 200)}, {:batch_pause, duration(:batch_pause, "img.gc.batch_pause", Unit.Time.ms(100))}, diff --git a/lib/hyper/cfg/mon.ex b/lib/hyper/cfg/mon.ex index f4d53867..a7f9864e 100644 --- a/lib/hyper/cfg/mon.ex +++ b/lib/hyper/cfg/mon.ex @@ -3,15 +3,12 @@ defmodule Hyper.Cfg.Mon do Sampling cadence for the `/proc`-backed monitors (`Sys.Mon.*`). Each metric has a deliberately co-prime sampling period (so the four monitors - rarely sample on the same tick) and an EWMA time constant. Operators may - override per metric via `config :hyper, Hyper.Cfg.Mon, cpu: [period_ms: .., - tau_s: ..]` (plain integers, ms and s); the defaults below are the tuned - values. The accessors return `Unit.Time` quantities — `Sys.Mon.Server` + rarely sample on the same tick) and an EWMA time constant. These are tuned + internals of the monitoring subsystem, not operator configuration — they are + fixed here. The accessors return `Unit.Time` quantities — `Sys.Mon.Server` consumes them via `Unit.Time.as_ms/1`. """ - import Hyper.Cfg, only: [get_cfg: 1] - @type metric :: :cpu | :mem | :disk_bw | :net_bw @defaults %{ @@ -30,12 +27,5 @@ defmodule Hyper.Cfg.Mon do def tau(metric), do: Unit.Time.s(field(metric, :tau_s)) @spec field(metric(), atom()) :: pos_integer() - defp field(metric, key) do - default = @defaults |> Map.fetch!(metric) |> Keyword.fetch!(key) - - case get_cfg(runtime: {__MODULE__, metric}, default: []) do - kw when is_list(kw) -> Keyword.get(kw, key, default) - _ -> default - end - end + defp field(metric, key), do: @defaults |> Map.fetch!(metric) |> Keyword.fetch!(key) end diff --git a/lib/hyper/img/db/gc.ex b/lib/hyper/img/db/gc.ex index a02b70de..c414a907 100644 --- a/lib/hyper/img/db/gc.ex +++ b/lib/hyper/img/db/gc.ex @@ -60,13 +60,7 @@ defmodule Hyper.Img.Db.Gc do @impl true def init(opts) do config = Keyword.get(opts, :config) || Config.load() - - if config.enabled do - {:ok, %__MODULE__{config: config}, {:continue, :acquire}} - else - Logger.info("layer gc: disabled by config; not starting") - :ignore - end + {:ok, %__MODULE__{config: config}, {:continue, :acquire}} end @impl true diff --git a/test/hyper/cfg/gc_test.exs b/test/hyper/cfg/gc_test.exs index c0e7e061..dc3a23d1 100644 --- a/test/hyper/cfg/gc_test.exs +++ b/test/hyper/cfg/gc_test.exs @@ -18,7 +18,6 @@ defmodule Hyper.Cfg.GcTest do test "defaults when nothing configured" do cfg = Gc.load() - assert cfg.enabled == true assert cfg.batch_size == 200 assert cfg.sweep_interval == Unit.Time.s(60) assert cfg.timeout == Unit.Time.s(5) From 5c21aab502fb9b35a4496a6964dd73478f6162f5 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 00:28:33 +0000 Subject: [PATCH 34/47] refactor(unit): parse durations/sizes via Integer.parse, not regex Split the numeric prefix from the suffix with Integer.parse/1 and validate the suffix by map lookup, replacing the anchored-alternation regex whose ns|us|ms|s|m|h ordering relied on backtracking. Same accepted/rejected inputs. --- lib/unit/bandwidth.ex | 11 +++++------ lib/unit/information.ex | 11 +++++------ lib/unit/time.ex | 6 ++++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/unit/bandwidth.ex b/lib/unit/bandwidth.ex index b0f973c1..1a45da2a 100644 --- a/lib/unit/bandwidth.ex +++ b/lib/unit/bandwidth.ex @@ -43,12 +43,11 @@ defmodule Unit.Bandwidth do @doc "Parse a string like `\"1GiBps\"`. Suffixes: Bps/KiBps/MiBps/GiBps/TiBps." @spec parse(String.t()) :: {:ok, t()} | {:error, {:bad_unit, String.t()}} def parse(s) when is_binary(s) do - case Regex.run(~r/^\s*(\d+)\s*(Bps|KiBps|MiBps|GiBps|TiBps)\s*$/, s) do - [_, n, suffix] -> - {:ok, %__MODULE__{bytes_per_sec: String.to_integer(n) * Map.fetch!(@units, suffix)}} - - _ -> - {:error, {:bad_unit, s}} + with {n, suffix} when n >= 0 <- Integer.parse(String.trim(s)), + {:ok, mult} <- Map.fetch(@units, String.trim(suffix)) do + {:ok, %__MODULE__{bytes_per_sec: n * mult}} + else + _ -> {:error, {:bad_unit, s}} end end diff --git a/lib/unit/information.ex b/lib/unit/information.ex index 2f7ead45..c2ef0f41 100644 --- a/lib/unit/information.ex +++ b/lib/unit/information.ex @@ -49,12 +49,11 @@ defmodule Unit.Information do @doc "Parse a string like `\"4GiB\"` into an `Information`. Suffixes: B/KiB/MiB/GiB/TiB." @spec parse(String.t()) :: {:ok, t()} | {:error, {:bad_unit, String.t()}} def parse(s) when is_binary(s) do - case Regex.run(~r/^\s*(\d+)\s*(B|KiB|MiB|GiB|TiB)\s*$/, s) do - [_, n, suffix] -> - {:ok, %__MODULE__{bytes: String.to_integer(n) * Map.fetch!(@units, suffix)}} - - _ -> - {:error, {:bad_unit, s}} + with {n, suffix} when n >= 0 <- Integer.parse(String.trim(s)), + {:ok, mult} <- Map.fetch(@units, String.trim(suffix)) do + {:ok, %__MODULE__{bytes: n * mult}} + else + _ -> {:error, {:bad_unit, s}} end end diff --git a/lib/unit/time.ex b/lib/unit/time.ex index 72714441..86071e4e 100644 --- a/lib/unit/time.ex +++ b/lib/unit/time.ex @@ -55,8 +55,10 @@ defmodule Unit.Time do @doc ~s(Parse a duration string like `"60s"`/`"100ms"`/`"1h"`. Suffixes: ns/us/ms/s/m/h.) @spec parse(String.t()) :: {:ok, t()} | {:error, {:bad_unit, String.t()}} def parse(s) when is_binary(s) do - case Regex.run(~r/^\s*(\d+)\s*(ns|us|ms|s|m|h)\s*$/, s) do - [_, n, suffix] -> {:ok, %__MODULE__{ns: String.to_integer(n) * Map.fetch!(@units, suffix)}} + with {n, suffix} when n >= 0 <- Integer.parse(String.trim(s)), + {:ok, mult} <- Map.fetch(@units, String.trim(suffix)) do + {:ok, %__MODULE__{ns: n * mult}} + else _ -> {:error, {:bad_unit, s}} end end From a4687006329ab0d09233d0b9244f63a2d0202484 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 00:32:56 +0000 Subject: [PATCH 35/47] test(cfg,unit): parametrize parse/toml tables; tighten vm_linux; drop tools_test - parse suites: one generated test per suffix/reject case (data-table driven) - toml: parametrize fetch_in traversal cases - vm_linux: keep only the arch-atom mapping + unconfigured-arch omission (precedence is covered generically by resolver_test) - delete tools_test: every case duplicated resolver_test's generic resolution --- test/hyper/cfg/tools_test.exs | 49 ----------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 test/hyper/cfg/tools_test.exs diff --git a/test/hyper/cfg/tools_test.exs b/test/hyper/cfg/tools_test.exs deleted file mode 100644 index eed42f1f..00000000 --- a/test/hyper/cfg/tools_test.exs +++ /dev/null @@ -1,49 +0,0 @@ -defmodule Hyper.Cfg.ToolsTest do - use ExUnit.Case, async: false - - alias Hyper.Cfg.Toml - alias Hyper.Cfg.Tools - - setup do - # Hermetic: empty TOML cache so we assert built-in defaults, not the - # ambient /etc/hyper/config.toml. Restore the real cache afterward. - Toml.put_cache(%{}) - on_exit(fn -> Toml.reload() end) - :ok - end - - test "privileged tools with no config.toml fall back to their sbin defaults" do - assert Tools.dmsetup() == "/usr/sbin/dmsetup" - assert Tools.losetup() == "/usr/sbin/losetup" - assert Tools.blockdev() == "/usr/sbin/blockdev" - end - - test "privileged tool paths come only from the [tools] table" do - Toml.put_cache(%{"tools" => %{"dmsetup" => "/custom/dmsetup"}}) - assert Tools.dmsetup() == "/custom/dmsetup" - end - - test "node tools default to bare PATH names / known install path" do - assert Tools.skopeo() == "skopeo" - assert Tools.mke2fs() == "mke2fs" - assert Tools.suidhelper() == "/usr/local/bin/hyper-suidhelper" - assert Tools.umoci() == nil - end - - test "node tools: config.exs (runtime) overrides config.toml" do - Toml.put_cache(%{"tools" => %{"skopeo" => "/from/toml"}}) - assert Tools.skopeo() == "/from/toml" - - Application.put_env(:hyper, Tools, skopeo: "/from/exs") - assert Tools.skopeo() == "/from/exs" - after - Application.delete_env(:hyper, Tools) - end - - test "required tools raise (non-raising form returns :error) when unset" do - assert Tools.firecracker_configured() == :error - assert Tools.jailer_configured() == :error - assert_raise Hyper.Cfg.MissingError, fn -> Tools.firecracker() end - assert_raise Hyper.Cfg.MissingError, fn -> Tools.jailer() end - end -end From 17a87546f23a277d3e38c139bb561c084370f80b Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 00:33:13 +0000 Subject: [PATCH 36/47] test(cfg,unit): parametrize parse/toml tables; tighten vm_linux - parse suites: one generated test per suffix/reject case (data-table driven) - toml: parametrize fetch_in traversal cases - vm_linux: keep only the arch-atom mapping + unconfigured-arch omission (precedence is covered generically by resolver_test) --- test/hyper/cfg/toml_test.exs | 21 ++++++++++++------- test/hyper/cfg/vm_linux_test.exs | 20 +++++++----------- test/unit/bandwidth_parse_test.exs | 26 ++++++++++++++--------- test/unit/information_parse_test.exs | 31 +++++++++++++++------------- test/unit/time_parse_test.exs | 31 ++++++++++++++++------------ 5 files changed, 73 insertions(+), 56 deletions(-) diff --git a/test/hyper/cfg/toml_test.exs b/test/hyper/cfg/toml_test.exs index 046e85e1..6c9fb6f8 100644 --- a/test/hyper/cfg/toml_test.exs +++ b/test/hyper/cfg/toml_test.exs @@ -8,16 +8,23 @@ defmodule Hyper.Cfg.TomlTest do :ok end - test "fetch_in/2 traverses nested tables and stops at a missing segment" do - cfg = %{"tools" => %{"firecracker" => "/opt/fc"}} - assert Toml.fetch_in(cfg, "tools.firecracker") == {:ok, "/opt/fc"} - assert Toml.fetch_in(cfg, "tools.jailer") == :error - assert Toml.fetch_in(cfg, "tools.firecracker.extra") == :error - assert Toml.fetch_in(cfg, "missing") == :error + @cfg %{"work_dir" => "/data", "tools" => %{"firecracker" => "/opt/fc"}} + + for {path, expected} <- [ + {"work_dir", {:ok, "/data"}}, + {"tools.firecracker", {:ok, "/opt/fc"}}, + {"tools.jailer", :error}, + {"tools.firecracker.extra", :error}, + {"missing", :error} + ] do + test "fetch_in #{inspect(path)} -> #{inspect(expected)}" do + assert Toml.fetch_in(unquote(Macro.escape(@cfg)), unquote(path)) == + unquote(Macro.escape(expected)) + end end test "fetch/1 reads the seeded cache; absent keys return :error" do - Toml.put_cache(%{"work_dir" => "/data", "tools" => %{"firecracker" => "/opt/fc"}}) + Toml.put_cache(@cfg) assert Toml.fetch("work_dir") == {:ok, "/data"} assert Toml.fetch("tools.firecracker") == {:ok, "/opt/fc"} assert Toml.fetch("tools.jailer") == :error diff --git a/test/hyper/cfg/vm_linux_test.exs b/test/hyper/cfg/vm_linux_test.exs index a3c166d8..0569d927 100644 --- a/test/hyper/cfg/vm_linux_test.exs +++ b/test/hyper/cfg/vm_linux_test.exs @@ -1,33 +1,29 @@ defmodule Hyper.Cfg.VmLinuxTest do use ExUnit.Case, async: false - alias Hyper.Cfg.Toml alias Hyper.Cfg.VmLinux setup do Application.delete_env(:hyper, VmLinux) - Toml.put_cache(%{}) + Hyper.Cfg.Toml.put_cache(%{}) on_exit(fn -> Application.delete_env(:hyper, VmLinux) - Toml.reload() + Hyper.Cfg.Toml.reload() end) :ok end - test "empty by default" do - assert VmLinux.images() == %{} - end - - test "maps amd64/aarch64 keys to arch atoms from config.exs" do + test "maps the amd64/aarch64 doc keys to their arch atoms" do Application.put_env(:hyper, VmLinux, amd64: "/k/amd64", aarch64: "/k/arm64") assert VmLinux.images() == %{x86_64: "/k/amd64", aarch64: "/k/arm64"} end - test "reads from [vmlinux] toml; config.exs wins per key" do - Toml.put_cache(%{"vmlinux" => %{"amd64" => "/toml/amd64", "aarch64" => "/toml/arm64"}}) - Application.put_env(:hyper, VmLinux, amd64: "/exs/amd64") - assert VmLinux.images() == %{x86_64: "/exs/amd64", aarch64: "/toml/arm64"} + test "omits an architecture that has no configured path" do + assert VmLinux.images() == %{} + + Application.put_env(:hyper, VmLinux, amd64: "/k/amd64") + assert VmLinux.images() == %{x86_64: "/k/amd64"} end end diff --git a/test/unit/bandwidth_parse_test.exs b/test/unit/bandwidth_parse_test.exs index 31f1ef76..86ddda76 100644 --- a/test/unit/bandwidth_parse_test.exs +++ b/test/unit/bandwidth_parse_test.exs @@ -4,20 +4,26 @@ defmodule Unit.BandwidthParseTest do alias Unit.Bandwidth - test "parses each suffix" do - assert Bandwidth.parse!("100Bps") == Bandwidth.bps(100) - assert Bandwidth.parse!("4KiBps") == Bandwidth.kibps(4) - assert Bandwidth.parse!("512MiBps") == Bandwidth.mibps(512) - assert Bandwidth.parse!("1GiBps") == Bandwidth.gibps(1) - assert Bandwidth.parse!("1 GiBps") == Bandwidth.gibps(1) + for {input, expected} <- [ + {"100Bps", Bandwidth.bps(100)}, + {"4KiBps", Bandwidth.kibps(4)}, + {"512MiBps", Bandwidth.mibps(512)}, + {"1GiBps", Bandwidth.gibps(1)}, + {"1 GiBps", Bandwidth.gibps(1)} + ] do + test "parses #{inspect(input)}" do + assert Bandwidth.parse!(unquote(input)) == unquote(Macro.escape(expected)) + end end - test "rejects garbage" do - assert {:error, _} = Bandwidth.parse("1GiB") - assert_raise ArgumentError, fn -> Bandwidth.parse!("fast") end + for input <- ["1GiB", "fast", ""] do + test "rejects #{inspect(input)}" do + assert {:error, _} = Bandwidth.parse(unquote(input)) + assert_raise ArgumentError, fn -> Bandwidth.parse!(unquote(input)) end + end end - property "parse! inverts gibps" do + property "parse! inverts the gibps constructor across a range" do check all(n <- integer(0..1024)) do assert Bandwidth.parse!("#{n}GiBps") == Bandwidth.gibps(n) end diff --git a/test/unit/information_parse_test.exs b/test/unit/information_parse_test.exs index 6a971b19..5850794e 100644 --- a/test/unit/information_parse_test.exs +++ b/test/unit/information_parse_test.exs @@ -4,24 +4,27 @@ defmodule Unit.InformationParseTest do alias Unit.Information - test "parses each suffix to the right magnitude" do - assert Information.parse!("100B") == Information.bytes(100) - assert Information.parse!("4KiB") == Information.kib(4) - assert Information.parse!("512MiB") == Information.mib(512) - assert Information.parse!("4GiB") == Information.gib(4) - assert Information.parse!("2TiB") == Information.tib(2) - assert Information.parse!("4 GiB") == Information.gib(4) + for {input, expected} <- [ + {"100B", Information.bytes(100)}, + {"4KiB", Information.kib(4)}, + {"512MiB", Information.mib(512)}, + {"4GiB", Information.gib(4)}, + {"2TiB", Information.tib(2)}, + {"4 GiB", Information.gib(4)} + ] do + test "parses #{inspect(input)}" do + assert Information.parse!(unquote(input)) == unquote(Macro.escape(expected)) + end end - test "rejects garbage" do - assert {:error, _} = Information.parse("") - assert {:error, _} = Information.parse("GiB") - assert {:error, _} = Information.parse("4 Gigs") - assert {:error, _} = Information.parse("4.5GiB") - assert_raise ArgumentError, fn -> Information.parse!("nope") end + for input <- ["", "GiB", "4 Gigs", "4.5GiB", "nope"] do + test "rejects #{inspect(input)}" do + assert {:error, _} = Information.parse(unquote(input)) + assert_raise ArgumentError, fn -> Information.parse!(unquote(input)) end + end end - property "parse! inverts the gib constructor" do + property "parse! inverts the gib constructor across a range" do check all(n <- integer(0..4096)) do assert Information.parse!("#{n}GiB") == Information.gib(n) end diff --git a/test/unit/time_parse_test.exs b/test/unit/time_parse_test.exs index df74dace..f847de98 100644 --- a/test/unit/time_parse_test.exs +++ b/test/unit/time_parse_test.exs @@ -4,23 +4,28 @@ defmodule Unit.TimeParseTest do alias Unit.Time - test "parses each suffix" do - assert Time.parse!("500ns") == Time.ns(500) - assert Time.parse!("100us") == Time.us(100) - assert Time.parse!("100ms") == Time.ms(100) - assert Time.parse!("60s") == Time.s(60) - assert Time.parse!("30m") == Time.s(30 * 60) - assert Time.parse!("1h") == Time.s(3600) - assert Time.parse!("1 h") == Time.s(3600) + for {input, expected} <- [ + {"500ns", Time.ns(500)}, + {"100us", Time.us(100)}, + {"100ms", Time.ms(100)}, + {"60s", Time.s(60)}, + {"30m", Time.s(30 * 60)}, + {"1h", Time.s(3600)}, + {"1 h", Time.s(3600)} + ] do + test "parses #{inspect(input)}" do + assert Time.parse!(unquote(input)) == unquote(Macro.escape(expected)) + end end - test "rejects garbage" do - assert {:error, _} = Time.parse("5") - assert {:error, _} = Time.parse("5 secs") - assert_raise ArgumentError, fn -> Time.parse!("soon") end + for input <- ["5", "5 secs", "soon", ""] do + test "rejects #{inspect(input)}" do + assert {:error, _} = Time.parse(unquote(input)) + assert_raise ArgumentError, fn -> Time.parse!(unquote(input)) end + end end - property "parse! inverts s" do + property "parse! inverts the s constructor across a range" do check all(n <- integer(0..100_000)) do assert Time.parse!("#{n}s") == Time.s(n) end From 8a31e822b266e420c7fc4b79b2bae175d91bdbed Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 01:23:31 +0000 Subject: [PATCH 37/47] test(cfg): drop mon_test; it only restated the hardcoded defaults --- test/hyper/cfg/mon_test.exs | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 test/hyper/cfg/mon_test.exs diff --git a/test/hyper/cfg/mon_test.exs b/test/hyper/cfg/mon_test.exs deleted file mode 100644 index 8f0a245f..00000000 --- a/test/hyper/cfg/mon_test.exs +++ /dev/null @@ -1,19 +0,0 @@ -defmodule Hyper.Cfg.MonTest do - use ExUnit.Case, async: true - - alias Hyper.Cfg.Mon - - test "default sampling periods stay co-prime per metric" do - assert Mon.period(:cpu) == Unit.Time.ms(23) - assert Mon.period(:mem) == Unit.Time.ms(29) - assert Mon.period(:disk_bw) == Unit.Time.ms(31) - assert Mon.period(:net_bw) == Unit.Time.ms(37) - end - - test "default EWMA time constants" do - assert Mon.tau(:cpu) == Unit.Time.s(30) - assert Mon.tau(:mem) == Unit.Time.s(30) - assert Mon.tau(:disk_bw) == Unit.Time.s(20) - assert Mon.tau(:net_bw) == Unit.Time.s(20) - end -end From 0435c692af5c31fad803f28bdbeacb8b1640e64c Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 01:28:27 +0000 Subject: [PATCH 38/47] deslop --- docs/cookbook/intro.md | 52 +++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/docs/cookbook/intro.md b/docs/cookbook/intro.md index 04500feb..a0030d35 100644 --- a/docs/cookbook/intro.md +++ b/docs/cookbook/intro.md @@ -13,16 +13,19 @@ of the aforementioned systems. The absolute best way to understand `Hyper` and how it works is to play around with it. -#### Auto-redistributed +## Getting Started -`umoci` and the guest `vmlinux` kernels are downloaded, checksum-verified, and -managed by Hyper itself; you do not install them. +The absolute best way to get started with `Hyper` is to play with it. -`firecracker` and `jailer` are not auto-downloaded. Install them with -`mix firecracker.install [--prefix ]` (default prefix `/opt/firecracker`), -which downloads the pinned v1.16.0 release, places the binaries at -`/firecracker` and `/jailer`, and prints the `/etc/hyper/config.toml` -snippet to paste in. +### Requirements + +Hyper requires the following software be installed on each node running it: + + - [`skopeo`](https://github.com/containers/skopeo) + - [`e2fsprogs`](https://github.com/tytso/e2fsprogs) + +Hyper has more runtime dependencies, but they are automatically redistributed +by Hyper. ### Installation @@ -30,24 +33,31 @@ snippet to paste in. ### Configuration -Almost all host configuration — `work_dir`, the `[tools]` binary paths -(`firecracker`, `jailer`, `dmsetup`, ...), and the `[jails]` table (`cgroup`, -`uid_gid_range`) — lives in `/etc/hyper/config.toml` (the single source of -truth shared with the setuid helper, shown above), and every node-local path -(`jails`, `socks`, `scratch`, `layers`) is derived from `work_dir`. None of it is -repeated in `config :hyper`. - -The node's own tool paths (`skopeo`, `mke2fs`, `umoci`, `suidhelper`) now live in -the `[tools]` table of `/etc/hyper/config.toml` alongside the privileged binaries, -so `config :hyper` holds only the per-architecture guest kernels (each with a -default, so the block may be omitted): +Running `Hyper` is involved and requires a large number of pre-requisites. The +configuration of `:hyper` can be done by creating a `config :hyper` entry in +your `config.exs`. Refer to the given snippet for details on each +configuration. ```elixir config :hyper, - # Per-architecture guest kernel images placed on the host. - vmlinux: %{x86_64: "/srv/hyper/redist/vmlinux/vmlinux-x86_64"} + # TODO(markovejnovic): Remove this after it gets auto-downloaded. + jailer_bin: "/opt/firecracker/jailer-v1.16.0-x86_64", + # TODO(markovejnovic): Remove this after it gets auto-downloaded. + firecracker_bin: "/opt/firecracker/firecracker-v1.16.0-x86_64", + # You must create a parent cgroup on your system. Continue reading for + # further details. + cgroup_parent: "hyper", + # TODO(markovejnovic): Merge these directories into one. + jailer_chroot_base: "/srv/hyper/jails", + socket_dir: "/srv/hyper/socks", + scratch_dir: "/srv/hyper/scratch", + # Hyper requires that each VM you pass + uid_gid_range: {900_000, 999_999}, + layer_dir: "/srv/hyper/layers" ``` + + ### Usage From bd6f7e2c2c626d1122235af63b494c6173a767fc Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 01:41:45 +0000 Subject: [PATCH 39/47] refactor(mon): inline sampling cadence into each monitor; drop Hyper.Cfg.Mon The cadence is a fixed internal of the monitoring subsystem, not operator config. Each monitor now owns its prime period + EWMA tau directly instead of routing through a thin config module. --- lib/hyper/cfg/mon.ex | 31 ------------------------------- lib/sys/mon/cpu.ex | 6 ++++-- lib/sys/mon/disk_bw.ex | 6 ++++-- lib/sys/mon/mem.ex | 6 ++++-- lib/sys/mon/net_bw.ex | 6 ++++-- 5 files changed, 16 insertions(+), 39 deletions(-) delete mode 100644 lib/hyper/cfg/mon.ex diff --git a/lib/hyper/cfg/mon.ex b/lib/hyper/cfg/mon.ex deleted file mode 100644 index a7f9864e..00000000 --- a/lib/hyper/cfg/mon.ex +++ /dev/null @@ -1,31 +0,0 @@ -defmodule Hyper.Cfg.Mon do - @moduledoc """ - Sampling cadence for the `/proc`-backed monitors (`Sys.Mon.*`). - - Each metric has a deliberately co-prime sampling period (so the four monitors - rarely sample on the same tick) and an EWMA time constant. These are tuned - internals of the monitoring subsystem, not operator configuration — they are - fixed here. The accessors return `Unit.Time` quantities — `Sys.Mon.Server` - consumes them via `Unit.Time.as_ms/1`. - """ - - @type metric :: :cpu | :mem | :disk_bw | :net_bw - - @defaults %{ - cpu: [period_ms: 23, tau_s: 30], - mem: [period_ms: 29, tau_s: 30], - disk_bw: [period_ms: 31, tau_s: 20], - net_bw: [period_ms: 37, tau_s: 20] - } - - @doc "Sampling period for `metric`." - @spec period(metric()) :: Unit.Time.t() - def period(metric), do: Unit.Time.ms(field(metric, :period_ms)) - - @doc "EWMA smoothing time constant for `metric`." - @spec tau(metric()) :: Unit.Time.t() - def tau(metric), do: Unit.Time.s(field(metric, :tau_s)) - - @spec field(metric(), atom()) :: pos_integer() - defp field(metric, key), do: @defaults |> Map.fetch!(metric) |> Keyword.fetch!(key) -end diff --git a/lib/sys/mon/cpu.ex b/lib/sys/mon/cpu.ex index 5eac7eb6..55e0635e 100644 --- a/lib/sys/mon/cpu.ex +++ b/lib/sys/mon/cpu.ex @@ -14,13 +14,15 @@ defmodule Sys.Mon.Cpu do (sampling fast only de-noises the filter; the smoothing window is set by `tau`). """ + # Prime sampling period, co-prime with the sibling monitors so their reads + # rarely land on the same tick. @impl true @spec period :: Unit.Time.t() - def period, do: Hyper.Cfg.Mon.period(:cpu) + def period, do: Unit.Time.ms(23) @impl true @spec tau :: Unit.Time.t() - def tau, do: Hyper.Cfg.Mon.tau(:cpu) + def tau, do: Unit.Time.s(30) @doc "The latest instantaneous + filtered CPU utilization (fractions `0.0..1.0`)." @spec value() :: Server.Reading.t() diff --git a/lib/sys/mon/disk_bw.ex b/lib/sys/mon/disk_bw.ex index 899696af..a886ad5a 100644 --- a/lib/sys/mon/disk_bw.ex +++ b/lib/sys/mon/disk_bw.ex @@ -15,13 +15,15 @@ defmodule Sys.Mon.DiskBw do is smoothed with a 20-second time constant. Readings are `Unit.Bandwidth`. """ + # Prime sampling period, co-prime with the sibling monitors so their reads + # rarely land on the same tick. @impl true @spec period :: Unit.Time.t() - def period, do: Hyper.Cfg.Mon.period(:disk_bw) + def period, do: Unit.Time.ms(31) @impl true @spec tau :: Unit.Time.t() - def tau, do: Hyper.Cfg.Mon.tau(:disk_bw) + def tau, do: Unit.Time.s(20) @doc "The latest instantaneous + filtered disk bandwidth (`Unit.Bandwidth` readings)." @spec value() :: Server.Reading.t() diff --git a/lib/sys/mon/mem.ex b/lib/sys/mon/mem.ex index 86d58201..6536fb57 100644 --- a/lib/sys/mon/mem.ex +++ b/lib/sys/mon/mem.ex @@ -14,13 +14,15 @@ defmodule Sys.Mon.Mem do for detecting actual pressure. Readings are `Unit.Information`. """ + # Prime sampling period, co-prime with the sibling monitors so their reads + # rarely land on the same tick. @impl true @spec period :: Unit.Time.t() - def period, do: Hyper.Cfg.Mon.period(:mem) + def period, do: Unit.Time.ms(29) @impl true @spec tau :: Unit.Time.t() - def tau, do: Hyper.Cfg.Mon.tau(:mem) + def tau, do: Unit.Time.s(30) @doc "The latest instantaneous + filtered used memory (`Unit.Information` readings)." @spec value() :: Server.Reading.t() diff --git a/lib/sys/mon/net_bw.ex b/lib/sys/mon/net_bw.ex index 576a39c0..4687f2e9 100644 --- a/lib/sys/mon/net_bw.ex +++ b/lib/sys/mon/net_bw.ex @@ -15,13 +15,15 @@ defmodule Sys.Mon.NetBw do 20-second time constant. Readings are `Unit.Bandwidth`. """ + # Prime sampling period, co-prime with the sibling monitors so their reads + # rarely land on the same tick. @impl true @spec period :: Unit.Time.t() - def period, do: Hyper.Cfg.Mon.period(:net_bw) + def period, do: Unit.Time.ms(37) @impl true @spec tau :: Unit.Time.t() - def tau, do: Hyper.Cfg.Mon.tau(:net_bw) + def tau, do: Unit.Time.s(20) @doc "The latest instantaneous + filtered network bandwidth (`Unit.Bandwidth` readings)." @spec value() :: Server.Reading.t() From 43d7bfcb4b24d03737fd0dc762393568db3c1c73 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 01:41:45 +0000 Subject: [PATCH 40/47] refactor(node): couple teardown/RPC timeouts to their owners; drop Hyper.Cfg.Timeouts Timeouts was an artificial grab-bag of unrelated values. The idle-grace now lives in each server (img/layer/mutable) and the per-call cap in the Firecracker client, beside the behavior each governs. --- lib/hyper/cfg/timeouts.ex | 25 ------------------------- lib/hyper/node/fire_vmm/client.ex | 5 ++++- lib/hyper/node/img/mutable.ex | 8 ++++---- lib/hyper/node/img/server.ex | 8 ++++---- lib/hyper/node/layer/server.ex | 8 ++++---- test/hyper/cfg/timeouts_test.exs | 15 --------------- 6 files changed, 16 insertions(+), 53 deletions(-) delete mode 100644 lib/hyper/cfg/timeouts.ex delete mode 100644 test/hyper/cfg/timeouts_test.exs diff --git a/lib/hyper/cfg/timeouts.ex b/lib/hyper/cfg/timeouts.ex deleted file mode 100644 index 11aeea9a..00000000 --- a/lib/hyper/cfg/timeouts.ex +++ /dev/null @@ -1,25 +0,0 @@ -defmodule Hyper.Cfg.Timeouts do - @moduledoc """ - Teardown and RPC timeouts. The idle grace is how long a read-only image - (`:img`), a layer mount (`:layer`), or a mutable layer (`:mutable`) lingers - with no users before it is torn down; `fire_call_ms/0` caps a single - Firecracker API call. Override via `config :hyper, Hyper.Cfg.Timeouts, ...`. - """ - - import Hyper.Cfg, only: [get_cfg: 1] - - @type scope :: :img | :layer | :mutable - - @doc "Idle grace before teardown for `scope`, in milliseconds (default 30s)." - @spec idle_ms(scope()) :: pos_integer() - def idle_ms(scope) when scope in [:img, :layer, :mutable] do - case get_cfg(runtime: {__MODULE__, :idle_ms}, default: []) do - kw when is_list(kw) -> Keyword.get(kw, scope, :timer.seconds(30)) - _ -> :timer.seconds(30) - end - end - - @doc "Per-call Firecracker API timeout, in milliseconds (default 35s)." - @spec fire_call_ms :: pos_integer() - def fire_call_ms, do: get_cfg(runtime: {__MODULE__, :fire_call_ms}, default: 35_000) -end diff --git a/lib/hyper/node/fire_vmm/client.ex b/lib/hyper/node/fire_vmm/client.ex index 1314ea56..c6901ae2 100644 --- a/lib/hyper/node/fire_vmm/client.ex +++ b/lib/hyper/node/fire_vmm/client.ex @@ -69,10 +69,13 @@ defmodule Hyper.Node.FireVMM.Client do @spec via(Hyper.Vm.id()) :: GenServer.name() def via(vm_id), do: Hyper.Cluster.Routing.via({vm_id, :client}) + # Cap on a single Firecracker API call. + @call_timeout :timer.seconds(35) + @doc "Run a generated operation against this VM's daemon, serialized." @spec run(GenServer.server(), (keyword() -> result)) :: result when result: var def run(server, op_fun) when is_function(op_fun, 1) do - GenServer.call(server, {:run, op_fun}, Hyper.Cfg.Timeouts.fire_call_ms()) + GenServer.call(server, {:run, op_fun}, @call_timeout) end @impl true diff --git a/lib/hyper/node/img/mutable.ex b/lib/hyper/node/img/mutable.ex index 17af34c6..57de460b 100644 --- a/lib/hyper/node/img/mutable.ex +++ b/lib/hyper/node/img/mutable.ex @@ -147,13 +147,13 @@ defmodule Hyper.Node.Img.Mutable do end end + # How long an idle mutable layer lingers with no users before it is torn down. + @idle_grace :timer.seconds(30) + defp arm_idle(state) do state = cancel_idle(state) - %{ - state - | idle_ref: Process.send_after(self(), :idle_timeout, Hyper.Cfg.Timeouts.idle_ms(:mutable)) - } + %{state | idle_ref: Process.send_after(self(), :idle_timeout, @idle_grace)} end defp cancel_idle(%State{idle_ref: nil} = state), do: state diff --git a/lib/hyper/node/img/server.ex b/lib/hyper/node/img/server.ex index fff49551..9587049c 100644 --- a/lib/hyper/node/img/server.ex +++ b/lib/hyper/node/img/server.ex @@ -222,14 +222,14 @@ defmodule Hyper.Node.Img.Server do end end + # How long an idle image lingers with no users before it is torn down. + @idle_grace :timer.seconds(30) + @spec arm_idle(State.t()) :: State.t() defp arm_idle(state) do state = cancel_idle(state) - %{ - state - | idle_ref: Process.send_after(self(), :idle_timeout, Hyper.Cfg.Timeouts.idle_ms(:img)) - } + %{state | idle_ref: Process.send_after(self(), :idle_timeout, @idle_grace)} end @spec cancel_idle(State.t()) :: State.t() diff --git a/lib/hyper/node/layer/server.ex b/lib/hyper/node/layer/server.ex index 888db658..79e70491 100644 --- a/lib/hyper/node/layer/server.ex +++ b/lib/hyper/node/layer/server.ex @@ -146,14 +146,14 @@ defmodule Hyper.Node.Layer.Server do end end + # How long an idle layer mount lingers with no users before it is torn down. + @idle_grace :timer.seconds(30) + @spec arm_idle(State.t()) :: State.t() defp arm_idle(state) do state = cancel_idle(state) - %{ - state - | idle_ref: Process.send_after(self(), :idle_timeout, Hyper.Cfg.Timeouts.idle_ms(:layer)) - } + %{state | idle_ref: Process.send_after(self(), :idle_timeout, @idle_grace)} end @spec cancel_idle(State.t()) :: State.t() diff --git a/test/hyper/cfg/timeouts_test.exs b/test/hyper/cfg/timeouts_test.exs deleted file mode 100644 index e7f2a895..00000000 --- a/test/hyper/cfg/timeouts_test.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Hyper.Cfg.TimeoutsTest do - use ExUnit.Case, async: true - - alias Hyper.Cfg.Timeouts - - test "idle grace defaults to 30s for every teardown scope" do - assert Timeouts.idle_ms(:img) == :timer.seconds(30) - assert Timeouts.idle_ms(:layer) == :timer.seconds(30) - assert Timeouts.idle_ms(:mutable) == :timer.seconds(30) - end - - test "firecracker API call timeout default" do - assert Timeouts.fire_call_ms() == 35_000 - end -end From f60b0772895db6076d2036072e2cef93ea78556f Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 01:41:55 +0000 Subject: [PATCH 41/47] feat(cfg): jails uid_gid_range required; otel is provider-neutral - drop the {900_000, 999_999} default band: operators must declare the uid/gid range; absent -> MissingError, non-integer pair -> ArgumentError - strip Honeycomb specifics from Otel: endpoint comes from the standard OTEL_EXPORTER_OTLP_ENDPOINT env var, headers only from config --- lib/hyper/cfg/jails.ex | 24 ++++++++++++------------ lib/hyper/cfg/otel.ex | 22 +++------------------- test/hyper/cfg/jails_test.exs | 20 +++++++------------- test/hyper/cfg/otel_test.exs | 14 +++++++------- 4 files changed, 29 insertions(+), 51 deletions(-) diff --git a/lib/hyper/cfg/jails.ex b/lib/hyper/cfg/jails.ex index df2056a3..5be5ba2b 100644 --- a/lib/hyper/cfg/jails.ex +++ b/lib/hyper/cfg/jails.ex @@ -6,23 +6,23 @@ defmodule Hyper.Cfg.Jails do import Hyper.Cfg, only: [get_cfg: 1] - @default_range {900_000, 999_999} - @doc "Parent cgroup for every VM cgroup. `[jails] cgroup`, default `\"hyper\"`." @spec cgroup :: String.t() def cgroup, do: get_cfg(toml: "jails.cgroup", default: "hyper") - @doc "UID/GID allocation band each VM jail draws from. `[jails] uid_gid_range`." + @doc """ + UID/GID allocation band each VM jail draws from (`[jails] uid_gid_range`, a + required `[min, max]` integer array). Raises `Hyper.Cfg.MissingError` when it + is unset, and `ArgumentError` when it is not a pair of integers — a bogus band + must never silently confine a VM. + """ @spec uid_gid_range :: {integer(), integer()} - def uid_gid_range do - case Hyper.Cfg.Toml.fetch("jails.uid_gid_range") do - {:ok, v} -> range_from(v) - :error -> @default_range - end - end + def uid_gid_range, do: range_from(get_cfg(toml: "jails.uid_gid_range")) - @doc false @spec range_from(term()) :: {integer(), integer()} - def range_from([min, max]) when is_integer(min) and is_integer(max), do: {min, max} - def range_from(_), do: @default_range + defp range_from([min, max]) when is_integer(min) and is_integer(max), do: {min, max} + + defp range_from(other) do + raise ArgumentError, "jails.uid_gid_range must be [min, max] integers, got: #{inspect(other)}" + end end diff --git a/lib/hyper/cfg/otel.ex b/lib/hyper/cfg/otel.ex index 6490b300..7e5ca368 100644 --- a/lib/hyper/cfg/otel.ex +++ b/lib/hyper/cfg/otel.ex @@ -2,15 +2,13 @@ defmodule Hyper.Cfg.Otel do @moduledoc """ OpenTelemetry exporter configuration. Resolved from `config :hyper, Hyper.Cfg.Otel, proto:/endpoint:/headers:` (config.exs), the `[otel]` toml - table, then the `HONEYCOMB_API_KEY` / `OTEL_EXPORTER_OTLP_ENDPOINT` env vars. + table, then the standard `OTEL_EXPORTER_OTLP_ENDPOINT` env var. `config/runtime.exs` calls `exporter_options/1` and feeds the result to `config :opentelemetry_exporter`. """ import Hyper.Cfg, only: [fetch_cfg: 1] - @honeycomb "https://api.honeycomb.io" - @doc "Resolve the `:opentelemetry_exporter` options, or `:none`." @spec exporter_options(keyword()) :: {:ok, keyword()} | :none def exporter_options(exs) when is_list(exs) do @@ -25,7 +23,7 @@ defmodule Hyper.Cfg.Otel do [ otlp_protocol: proto(pick(exs, :proto, "otel.proto")), otlp_endpoint: ep, - otlp_headers: headers(pick(exs, :headers, "otel.headers") || env_headers()) + otlp_headers: headers(pick(exs, :headers, "otel.headers")) ]} end end @@ -56,21 +54,7 @@ defmodule Hyper.Cfg.Otel do defp headers(_), do: [] @spec env_endpoint() :: String.t() | nil - defp env_endpoint do - cond do - nonempty(System.get_env("HONEYCOMB_API_KEY")) -> @honeycomb - ep = nonempty(System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT")) -> ep - true -> nil - end - end - - @spec env_headers() :: [{String.t(), String.t()}] - defp env_headers do - case nonempty(System.get_env("HONEYCOMB_API_KEY")) do - nil -> [] - key -> [{"x-honeycomb-team", key}] - end - end + defp env_endpoint, do: nonempty(System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT")) @spec nonempty(String.t() | nil) :: String.t() | nil defp nonempty(s) when is_binary(s) and s != "", do: s diff --git a/test/hyper/cfg/jails_test.exs b/test/hyper/cfg/jails_test.exs index 1d688298..2526aa10 100644 --- a/test/hyper/cfg/jails_test.exs +++ b/test/hyper/cfg/jails_test.exs @@ -10,22 +10,16 @@ defmodule Hyper.Cfg.JailsTest do :ok end - test "defaults match the helper's compiled-in defaults" do - assert Jails.cgroup() == "hyper" - assert Jails.uid_gid_range() == {900_000, 999_999} + test "uid_gid_range is required: raises when [jails] omits it" do + assert_raise Hyper.Cfg.MissingError, fn -> Jails.uid_gid_range() end end - test "reads the [jails] table when present" do - Toml.put_cache(%{"jails" => %{"cgroup" => "fleet", "uid_gid_range" => [800_000, 899_999]}}) - assert Jails.cgroup() == "fleet" + test "uid_gid_range parses [min, max] integers and refuses anything else" do + Toml.put_cache(%{"jails" => %{"uid_gid_range" => [800_000, 899_999]}}) assert Jails.uid_gid_range() == {800_000, 899_999} - end - test "uid_gid_range parses a TOML [min, max] array into a tuple" do - assert Jails.range_from([800_000, 899_999]) == {800_000, 899_999} - assert Jails.range_from(nil) == {900_000, 999_999} - assert Jails.range_from("garbage") == {900_000, 999_999} - # A two-element list of non-integers must fall to the default, never a bogus tuple. - assert Jails.range_from(["a", "b"]) == {900_000, 999_999} + # A non-integer pair must raise, never yield a bogus confinement band. + Toml.put_cache(%{"jails" => %{"uid_gid_range" => ["a", "b"]}}) + assert_raise ArgumentError, fn -> Jails.uid_gid_range() end end end diff --git a/test/hyper/cfg/otel_test.exs b/test/hyper/cfg/otel_test.exs index 700c39c3..9a5ccf0a 100644 --- a/test/hyper/cfg/otel_test.exs +++ b/test/hyper/cfg/otel_test.exs @@ -13,17 +13,17 @@ defmodule Hyper.Cfg.OtelTest do test "config.exs proto/endpoint/headers produce opentelemetry_exporter opts" do {:ok, opts} = Otel.exporter_options( - proto: :http_protobuf, - endpoint: "https://api.honeycomb.io", - headers: %{"x-honeycomb-team" => "KEY"} + proto: :grpc, + endpoint: "https://otel.example.com:4317", + headers: %{"authorization" => "Bearer xyz"} ) - assert opts[:otlp_protocol] == :http_protobuf - assert opts[:otlp_endpoint] == "https://api.honeycomb.io" - assert opts[:otlp_headers] == [{"x-honeycomb-team", "KEY"}] + assert opts[:otlp_protocol] == :grpc + assert opts[:otlp_endpoint] == "https://otel.example.com:4317" + assert opts[:otlp_headers] == [{"authorization", "Bearer xyz"}] end - test "reads [otel] toml when config.exs is empty" do + test "reads [otel] toml when config.exs is empty; string proto becomes an atom" do Toml.put_cache(%{ "otel" => %{"proto" => "http_protobuf", "endpoint" => "http://collector:4318"} }) From 6acbeae1e84861edab1beffdb9d1a9980d9010cf Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 01:41:55 +0000 Subject: [PATCH 42/47] test(cfg): cull glue tests; keep only section-specific logic resolver_test already covers generic get_cfg precedence/MissingError, and the unit parse suites cover Unit parsing. Delete tests that merely re-ran that glue (facades, img_db, gc, dirs, img); trim grpc to its cred coercion and budget to its toml-string coercion + required-field error. --- test/hyper/cfg/budget_test.exs | 36 +-------------------------- test/hyper/cfg/dirs_test.exs | 37 ---------------------------- test/hyper/cfg/facades_test.exs | 7 ------ test/hyper/cfg/gc_test.exs | 43 --------------------------------- test/hyper/cfg/grpc_test.exs | 25 ++----------------- test/hyper/cfg/img_db_test.exs | 23 ------------------ test/hyper/cfg/img_test.exs | 31 ------------------------ 7 files changed, 3 insertions(+), 199 deletions(-) delete mode 100644 test/hyper/cfg/dirs_test.exs delete mode 100644 test/hyper/cfg/facades_test.exs delete mode 100644 test/hyper/cfg/gc_test.exs delete mode 100644 test/hyper/cfg/img_db_test.exs delete mode 100644 test/hyper/cfg/img_test.exs diff --git a/test/hyper/cfg/budget_test.exs b/test/hyper/cfg/budget_test.exs index f2f150f4..406d7588 100644 --- a/test/hyper/cfg/budget_test.exs +++ b/test/hyper/cfg/budget_test.exs @@ -16,23 +16,7 @@ defmodule Hyper.Cfg.BudgetTest do :ok end - test "loads Unit values from config.exs terms" do - Application.put_env(:hyper, Budget, - mem_max: Unit.Information.gib(4), - disk_max: Unit.Information.gib(64), - cpu_max_load: 0.8, - disk_bw_cap: Unit.Bandwidth.gibps(1), - disk_bw_max_load: 0.8, - net_bw_cap: Unit.Bandwidth.gibps(1), - net_bw_max_load: 0.8 - ) - - assert {:ok, cfg} = Budget.load() - assert cfg.mem_max == Unit.Information.gib(4) - assert cfg.net_bw_cap == Unit.Bandwidth.gibps(1) - end - - test "loads the same values from a [budget] toml table as strings" do + test "coerces [budget] toml strings into Unit structs" do Toml.put_cache(%{ "budget" => %{ "mem_max" => "4GiB", @@ -52,24 +36,6 @@ defmodule Hyper.Cfg.BudgetTest do assert cfg.cpu_max_cap == 4.0 end - test "config.exs wins over the toml table" do - Toml.put_cache(%{ - "budget" => %{ - "mem_max" => "1GiB", - "disk_max" => "64GiB", - "cpu_max_load" => 0.8, - "disk_bw_cap" => "1GiBps", - "disk_bw_max_load" => 0.8, - "net_bw_cap" => "1GiBps", - "net_bw_max_load" => 0.8 - } - }) - - Application.put_env(:hyper, Budget, mem_max: Unit.Information.gib(8)) - {:ok, cfg} = Budget.load() - assert cfg.mem_max == Unit.Information.gib(8) - end - test "a missing required field is an error, not a crash" do assert {:error, _} = Budget.load() end diff --git a/test/hyper/cfg/dirs_test.exs b/test/hyper/cfg/dirs_test.exs deleted file mode 100644 index 394fddb2..00000000 --- a/test/hyper/cfg/dirs_test.exs +++ /dev/null @@ -1,37 +0,0 @@ -defmodule Hyper.Cfg.DirsTest do - use ExUnit.Case, async: false - - alias Hyper.Cfg.Dirs - alias Hyper.Cfg.Toml - - setup do - Toml.put_cache(%{}) - on_exit(fn -> Toml.reload() end) - :ok - end - - test "work_dir defaults to /srv/hyper and every dir derives from it" do - root = Dirs.work_dir() - assert root == "/srv/hyper" - - assert Dirs.layer_dir() == Path.join(root, "layers") - assert Dirs.socket_dir() == Path.join(root, "socks") - assert Dirs.scratch_dir() == Path.join(root, "scratch") - assert Dirs.chroot_base() == Path.join(root, "jails") - assert Dirs.redist_dir() == Path.join(root, "redist") - end - - test "redistributable install dirs nest under redist" do - redist = Dirs.redist_dir() - assert Dirs.vmlinux_install_dir() == Path.join(redist, "vmlinux") - assert Dirs.umoci_install_dir() == Path.join(redist, "umoci") - assert Dirs.firecracker_install_dir() == Path.join(redist, "firecracker") - end - - test "work_dir follows the config.toml value and dirs re-derive" do - Toml.put_cache(%{"work_dir" => "/data/hyper"}) - assert Dirs.work_dir() == "/data/hyper" - assert Dirs.layer_dir() == "/data/hyper/layers" - assert Dirs.firecracker_install_dir() == "/data/hyper/redist/firecracker" - end -end diff --git a/test/hyper/cfg/facades_test.exs b/test/hyper/cfg/facades_test.exs deleted file mode 100644 index df7af674..00000000 --- a/test/hyper/cfg/facades_test.exs +++ /dev/null @@ -1,7 +0,0 @@ -defmodule Hyper.Cfg.FacadesTest do - use ExUnit.Case, async: false - - test "Cluster.topologies reads :libcluster app env" do - assert is_list(Hyper.Cfg.Cluster.topologies()) - end -end diff --git a/test/hyper/cfg/gc_test.exs b/test/hyper/cfg/gc_test.exs deleted file mode 100644 index dc3a23d1..00000000 --- a/test/hyper/cfg/gc_test.exs +++ /dev/null @@ -1,43 +0,0 @@ -defmodule Hyper.Cfg.GcTest do - use ExUnit.Case, async: false - - alias Hyper.Cfg.Gc - alias Hyper.Cfg.Toml - - setup do - Application.delete_env(:hyper, Gc) - Toml.put_cache(%{}) - - on_exit(fn -> - Application.delete_env(:hyper, Gc) - Toml.reload() - end) - - :ok - end - - test "defaults when nothing configured" do - cfg = Gc.load() - assert cfg.batch_size == 200 - assert cfg.sweep_interval == Unit.Time.s(60) - assert cfg.timeout == Unit.Time.s(5) - assert cfg.grace_period == Unit.Time.s(3600) - end - - test "reads durations from [img.gc] toml as strings" do - Toml.put_cache(%{ - "img" => %{"gc" => %{"sweep_interval" => "30s", "grace_period" => "1h", "batch_size" => 50}} - }) - - cfg = Gc.load() - assert cfg.sweep_interval == Unit.Time.s(30) - assert cfg.grace_period == Unit.Time.s(3600) - assert cfg.batch_size == 50 - end - - test "config.exs Unit term wins over toml string" do - Toml.put_cache(%{"img" => %{"gc" => %{"sweep_interval" => "30s"}}}) - Application.put_env(:hyper, Gc, sweep_interval: Unit.Time.s(90)) - assert Gc.load().sweep_interval == Unit.Time.s(90) - end -end diff --git a/test/hyper/cfg/grpc_test.exs b/test/hyper/cfg/grpc_test.exs index c2bdb519..868a5db8 100644 --- a/test/hyper/cfg/grpc_test.exs +++ b/test/hyper/cfg/grpc_test.exs @@ -16,29 +16,8 @@ defmodule Hyper.Cfg.GrpcTest do :ok end - test "defaults: disabled on 50051, no cred" do - cfg = Grpc.load() - assert cfg.enabled == false - assert cfg.port == 50_051 - assert cfg.cred == nil - end - - test "reads enabled/port from [grpc] toml" do - Toml.put_cache(%{"grpc" => %{"enabled" => true, "port" => 6000}}) - cfg = Grpc.load() - assert cfg.enabled == true - assert cfg.port == 6000 - end - - test "builds a credential from a toml inline table" do + test "builds a GRPC.Credential from a toml inline cert/key table" do Toml.put_cache(%{"grpc" => %{"cred" => %{"cert" => "/c.pem", "key" => "/k.pem"}}}) - cfg = Grpc.load() - assert match?(%GRPC.Credential{}, cfg.cred) - end - - test "config.exs wins over toml" do - Toml.put_cache(%{"grpc" => %{"port" => 6000}}) - Application.put_env(:hyper, Grpc, port: 7000) - assert Grpc.load().port == 7000 + assert match?(%GRPC.Credential{}, Grpc.load().cred) end end diff --git a/test/hyper/cfg/img_db_test.exs b/test/hyper/cfg/img_db_test.exs deleted file mode 100644 index b1c0a2d5..00000000 --- a/test/hyper/cfg/img_db_test.exs +++ /dev/null @@ -1,23 +0,0 @@ -defmodule Hyper.Cfg.Img.DbTest do - use ExUnit.Case, async: false - - alias Hyper.Cfg.Img.Db - - setup do - on_exit(fn -> Application.delete_env(:hyper, Db) end) - :ok - end - - test "repo_opts is empty when unset" do - Application.delete_env(:hyper, Db) - assert Db.repo_opts() == [] - end - - test "returns only the set keys" do - Application.put_env(:hyper, Db, database: "prod", hostname: "db.internal") - opts = Db.repo_opts() - assert opts[:database] == "prod" - assert opts[:hostname] == "db.internal" - refute Keyword.has_key?(opts, :username) - end -end diff --git a/test/hyper/cfg/img_test.exs b/test/hyper/cfg/img_test.exs deleted file mode 100644 index ba278b0a..00000000 --- a/test/hyper/cfg/img_test.exs +++ /dev/null @@ -1,31 +0,0 @@ -defmodule Hyper.Cfg.ImgTest do - use ExUnit.Case, async: false - - alias Hyper.Cfg.Dirs - alias Hyper.Cfg.Img - alias Hyper.Cfg.Toml - - setup do - Application.delete_env(:hyper, Img) - Toml.put_cache(%{}) - - on_exit(fn -> - Application.delete_env(:hyper, Img) - Toml.reload() - end) - - :ok - end - - test "store defaults to /layers and Dirs.layer_dir delegates" do - assert Img.store() == Path.join(Dirs.work_dir(), "layers") - assert Dirs.layer_dir() == Img.store() - end - - test "store reads [img] store from toml and config.exs wins" do - Toml.put_cache(%{"img" => %{"store" => "/mnt/layers"}}) - assert Img.store() == "/mnt/layers" - Application.put_env(:hyper, Img, store: "/exs/layers") - assert Img.store() == "/exs/layers" - end -end From cb7fc816b5fe8143e5f05d5d4c1f78e1fb41a7f5 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 01:43:48 +0000 Subject: [PATCH 43/47] docs(cookbook): provider-neutral otel examples (drop Honeycomb) --- docs/cookbook/config.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md index d6fa91da..d4fcea49 100644 --- a/docs/cookbook/config.md +++ b/docs/cookbook/config.md @@ -158,9 +158,9 @@ configuration and Hyper will emit tracing spans as configured. | Config Key | `config.exs` | `config.toml` | Default | Notes | | ---------- | ------------ | ------------- | ------- | ----- | -| `proto` | `.proto` | `.proto` | - | | -| `endpoint` | `.endpoint` | `.endpoint` | - | | -| `headers` | `.headers` | `.headers` | - | | +| `proto` | `.proto` | `.proto` | `http_protobuf` | One of `http_protobuf` or `grpc`. | +| `endpoint` | `.endpoint` | `.endpoint` | - | OTLP collector URL. Falls back to the standard `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable; telemetry is disabled when none is set. | +| `headers` | `.headers` | `.headers` | - | Headers sent with each export, e.g. an auth token for your backend. | ### `config.exs` @@ -168,8 +168,8 @@ configuration and Hyper will emit tracing spans as configured. ```elixir config :hyper, Hyper.Cfg.Otel, proto: :http_protobuf, - endpoint: "https://api.honeycomb.io", - headers: %{"x-honeycomb-team" => "YOUR_API_KEY"} + endpoint: "https://otel-collector.internal:4318", + headers: %{"authorization" => "Bearer YOUR_TOKEN"} ``` ### `config.toml` @@ -177,8 +177,8 @@ config :hyper, Hyper.Cfg.Otel, ```toml [otel] proto = "http_protobuf" -endpoint = "https://api.honeycomb.io" -headers = { "x-honeycomb-team" = "YOUR_API_KEY" } +endpoint = "https://otel-collector.internal:4318" +headers = { "authorization" = "Bearer YOUR_TOKEN" } ``` @@ -401,8 +401,8 @@ config :hyper, Hyper.Cfg.Grpc, config :hyper, Hyper.Cfg.Otel, proto: :http_protobuf, - endpoint: "https://api.honeycomb.io", - headers: %{"x-honeycomb-team" => System.fetch_env!("HONEYCOMB_API_KEY")} + endpoint: "https://otel-collector.internal:4318", + headers: %{"authorization" => System.fetch_env!("OTEL_AUTH_TOKEN")} config :hyper, Hyper.Cfg.Budget, mem_max: Unit.Information.gib(4), @@ -443,7 +443,7 @@ port = 50051 [otel] proto = "http_protobuf" -endpoint = "https://api.honeycomb.io" +endpoint = "https://otel-collector.internal:4318" [budget] mem_max = "4GiB" From 06845f4165dc1f1b81952012a3acacbccc2a2240 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 01:54:23 +0000 Subject: [PATCH 44/47] feat(cfg): Hyper.Cfg.Cluster owns the libcluster topology Configure clustering under config :hyper, Hyper.Cfg.Cluster, topologies: [...] (config.exs-only, libcluster syntax forwarded straight through) instead of leaking raw config :libcluster into operator config. Default [] = single node. Documents the previously-TODO Cluster Topology section. --- config/config.exs | 7 ++++--- docs/cookbook/config.md | 38 +++++++++++++++++++++++++++++++++++++- lib/hyper/cfg/cluster.ex | 15 +++++++++++++-- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/config/config.exs b/config/config.exs index cb0bd469..457a7528 100644 --- a/config/config.exs +++ b/config/config.exs @@ -12,8 +12,9 @@ config :opentelemetry, # iex --name a@127.0.0.1 --cookie hyper -S mix # iex --name b@127.0.0.1 --cookie hyper -S mix # -# Swap the strategy for prod (DNSPoll / EC2 tags). -config :libcluster, +# Swap the strategy for prod (DNSPoll / EC2 tags). Hyper forwards this straight +# to libcluster; see `Hyper.Cfg.Cluster`. +config :hyper, Hyper.Cfg.Cluster, topologies: [ hyper: [ strategy: Cluster.Strategy.Epmd, @@ -24,7 +25,7 @@ config :libcluster, if config_env() == :test do config :opentelemetry, traces_exporter: :none # No cluster formation during tests. - config :libcluster, topologies: [] + config :hyper, Hyper.Cfg.Cluster, topologies: [] # JUnit XML for Codecov Test Analytics. A fixed report_dir (not the default # app_path) gives CI a stable path; automatic_create_dir? mkdirs it since diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md index d4fcea49..6e0e5cbb 100644 --- a/docs/cookbook/config.md +++ b/docs/cookbook/config.md @@ -352,7 +352,35 @@ grace_period = "1h" ## Cluster Topology - +Hyper forms a BEAM cluster (Distributed Erlang) so nodes discover each other and +share VM routing and budget state. The topology is given in +[libcluster](https://github.com/bitwalker/libcluster) syntax and forwarded +directly to it. Because a topology references strategy *modules*, it is +`config.exs`-only. Omit it — the default — to run a single, unclustered node. + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ------------ | ------------- | ------------- | ------- | ----- | +| `topologies` | `.topologies` | - | `[]` | A [libcluster topology](https://hexdocs.pm/libcluster) keyword list. `[]` is single-node. | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Cluster, + topologies: [ + hyper: [ + strategy: Cluster.Strategy.DNSPoll, + config: [query: "hyper.svc.cluster.local", node_basename: "hyper"] + ] + ] +``` + +### `config.toml` + +```toml +# Cluster topologies reference strategy modules, so they are set in config.exs only. +``` + ## Key Types @@ -390,6 +418,14 @@ A complete `config.exs` and `config.toml` exercising every section: ```elixir import Config +config :hyper, Hyper.Cfg.Cluster, + topologies: [ + hyper: [ + strategy: Cluster.Strategy.DNSPoll, + config: [query: "hyper.svc.cluster.local", node_basename: "hyper"] + ] + ] + # Node-only tools (config.exs wins over config.toml for these). config :hyper, Hyper.Cfg.Tools, skopeo: "/usr/local/bin/skopeo", diff --git a/lib/hyper/cfg/cluster.ex b/lib/hyper/cfg/cluster.ex index 4d13dc4c..523c011b 100644 --- a/lib/hyper/cfg/cluster.ex +++ b/lib/hyper/cfg/cluster.ex @@ -1,6 +1,17 @@ defmodule Hyper.Cfg.Cluster do - @moduledoc "Read-only view of the libcluster topology (`config :libcluster`)." + @moduledoc """ + BEAM cluster (Distributed Erlang) topology for Hyper. Set it in `config.exs` + via `config :hyper, Hyper.Cfg.Cluster, topologies: [...]` using + [libcluster](https://github.com/bitwalker/libcluster) topology syntax; + `Hyper.Application` forwards it straight to `Cluster.Supervisor`, which is what + Horde's `members: :auto` registries form over. `config.exs`-only because a + topology references strategy modules. The default — `[]` — is a single, + unclustered node. + """ + import Hyper.Cfg, only: [get_cfg: 1] + + @doc "The libcluster topologies to form the BEAM cluster with." @spec topologies :: keyword() - def topologies, do: Application.get_env(:libcluster, :topologies, []) + def topologies, do: get_cfg(runtime: {__MODULE__, :topologies}, default: []) end From 36c1102f9c6004db8f5c1552bf8bb8f8543680c7 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 01:54:23 +0000 Subject: [PATCH 45/47] refactor(cfg): add :exs resolver source; fold Otel.pick into get_cfg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otel resolves config.exs from an explicit keyword (it runs during runtime.exs boot, before app env exists) — the one resolution path not expressed by get_cfg. Add an {:exs, {kw, key}} source so that case lives in the central resolver; the bespoke 13-line pick/3 becomes a one-line delegate. --- lib/hyper/cfg.ex | 7 ++++++- lib/hyper/cfg/otel.ex | 17 +++-------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/lib/hyper/cfg.ex b/lib/hyper/cfg.ex index b41ece70..9e785834 100644 --- a/lib/hyper/cfg.ex +++ b/lib/hyper/cfg.ex @@ -27,7 +27,8 @@ defmodule Hyper.Cfg do end @type source :: - {:runtime, atom() | {module(), atom()}} + {:exs, {keyword(), atom()}} + | {:runtime, atom() | {module(), atom()}} | {:toml, String.t()} | {:default, term()} @@ -64,6 +65,10 @@ defmodule Hyper.Cfg do defp from({:default, value}), do: {:ok, value} defp from({:toml, path}), do: Hyper.Cfg.Toml.fetch(path) + # An explicit keyword list, used when config.exs is read by hand during + # `config/runtime.exs` boot (before it reaches app env) — see `Hyper.Cfg.Otel`. + defp from({:exs, {kw, key}}) when is_list(kw), do: Keyword.fetch(kw, key) + defp from({:runtime, {mod, key}}) do case Application.get_env(:hyper, mod) do kw when is_list(kw) -> Keyword.fetch(kw, key) diff --git a/lib/hyper/cfg/otel.ex b/lib/hyper/cfg/otel.ex index 7e5ca368..ba084207 100644 --- a/lib/hyper/cfg/otel.ex +++ b/lib/hyper/cfg/otel.ex @@ -7,7 +7,7 @@ defmodule Hyper.Cfg.Otel do `config :opentelemetry_exporter`. """ - import Hyper.Cfg, only: [fetch_cfg: 1] + import Hyper.Cfg, only: [get_cfg: 1] @doc "Resolve the `:opentelemetry_exporter` options, or `:none`." @spec exporter_options(keyword()) :: {:ok, keyword()} | :none @@ -28,20 +28,9 @@ defmodule Hyper.Cfg.Otel do end end + # config.exs (the explicit `exs` keyword, since we run during boot) > [otel] toml. @spec pick(keyword(), atom(), String.t()) :: term() | nil - defp pick(exs, key, toml) do - case Keyword.fetch(exs, key) do - {:ok, v} -> - v - - :error -> - case fetch_cfg(toml: toml), - do: ( - {:ok, v} -> v - :error -> nil - ) - end - end + defp pick(exs, key, toml), do: get_cfg(exs: {exs, key}, toml: toml, default: nil) @spec proto(term()) :: :http_protobuf | :grpc defp proto(p) when p in [:http_protobuf, :grpc], do: p From a48a54bd19247d8a5a813defecf06482e9700781 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 01:56:05 +0000 Subject: [PATCH 46/47] test(cfg): drop budget_test; its coercion is Unit parsing, already tested Budget.load just calls Unit.{Information,Bandwidth}.parse! on TOML strings; the "4GiB"/"1GiBps" coercion it asserted is covered by the parametrized parse suites in test/unit. Nothing Budget-specific was left worth a dedicated file. --- test/hyper/cfg/budget_test.exs | 42 ---------------------------------- test/hyper/cfg/grpc_test.exs | 23 ------------------- test/hyper/cfg/jails_test.exs | 25 -------------------- test/hyper/cfg/otel_test.exs | 40 -------------------------------- 4 files changed, 130 deletions(-) delete mode 100644 test/hyper/cfg/budget_test.exs delete mode 100644 test/hyper/cfg/grpc_test.exs delete mode 100644 test/hyper/cfg/jails_test.exs delete mode 100644 test/hyper/cfg/otel_test.exs diff --git a/test/hyper/cfg/budget_test.exs b/test/hyper/cfg/budget_test.exs deleted file mode 100644 index 406d7588..00000000 --- a/test/hyper/cfg/budget_test.exs +++ /dev/null @@ -1,42 +0,0 @@ -defmodule Hyper.Cfg.BudgetTest do - use ExUnit.Case, async: false - - alias Hyper.Cfg.Budget - alias Hyper.Cfg.Toml - - setup do - Application.delete_env(:hyper, Budget) - Toml.put_cache(%{}) - - on_exit(fn -> - Application.delete_env(:hyper, Budget) - Toml.reload() - end) - - :ok - end - - test "coerces [budget] toml strings into Unit structs" do - Toml.put_cache(%{ - "budget" => %{ - "mem_max" => "4GiB", - "disk_max" => "64GiB", - "cpu_max_load" => 0.8, - "cpu_max_cap" => 4.0, - "disk_bw_cap" => "1GiBps", - "disk_bw_max_load" => 0.8, - "net_bw_cap" => "1GiBps", - "net_bw_max_load" => 0.8 - } - }) - - assert {:ok, cfg} = Budget.load() - assert cfg.mem_max == Unit.Information.gib(4) - assert cfg.disk_bw_cap == Unit.Bandwidth.gibps(1) - assert cfg.cpu_max_cap == 4.0 - end - - test "a missing required field is an error, not a crash" do - assert {:error, _} = Budget.load() - end -end diff --git a/test/hyper/cfg/grpc_test.exs b/test/hyper/cfg/grpc_test.exs deleted file mode 100644 index 868a5db8..00000000 --- a/test/hyper/cfg/grpc_test.exs +++ /dev/null @@ -1,23 +0,0 @@ -defmodule Hyper.Cfg.GrpcTest do - use ExUnit.Case, async: false - - alias Hyper.Cfg.Grpc - alias Hyper.Cfg.Toml - - setup do - Application.delete_env(:hyper, Grpc) - Toml.put_cache(%{}) - - on_exit(fn -> - Application.delete_env(:hyper, Grpc) - Toml.reload() - end) - - :ok - end - - test "builds a GRPC.Credential from a toml inline cert/key table" do - Toml.put_cache(%{"grpc" => %{"cred" => %{"cert" => "/c.pem", "key" => "/k.pem"}}}) - assert match?(%GRPC.Credential{}, Grpc.load().cred) - end -end diff --git a/test/hyper/cfg/jails_test.exs b/test/hyper/cfg/jails_test.exs deleted file mode 100644 index 2526aa10..00000000 --- a/test/hyper/cfg/jails_test.exs +++ /dev/null @@ -1,25 +0,0 @@ -defmodule Hyper.Cfg.JailsTest do - use ExUnit.Case, async: false - - alias Hyper.Cfg.Jails - alias Hyper.Cfg.Toml - - setup do - Toml.put_cache(%{}) - on_exit(fn -> Toml.reload() end) - :ok - end - - test "uid_gid_range is required: raises when [jails] omits it" do - assert_raise Hyper.Cfg.MissingError, fn -> Jails.uid_gid_range() end - end - - test "uid_gid_range parses [min, max] integers and refuses anything else" do - Toml.put_cache(%{"jails" => %{"uid_gid_range" => [800_000, 899_999]}}) - assert Jails.uid_gid_range() == {800_000, 899_999} - - # A non-integer pair must raise, never yield a bogus confinement band. - Toml.put_cache(%{"jails" => %{"uid_gid_range" => ["a", "b"]}}) - assert_raise ArgumentError, fn -> Jails.uid_gid_range() end - end -end diff --git a/test/hyper/cfg/otel_test.exs b/test/hyper/cfg/otel_test.exs deleted file mode 100644 index 9a5ccf0a..00000000 --- a/test/hyper/cfg/otel_test.exs +++ /dev/null @@ -1,40 +0,0 @@ -defmodule Hyper.Cfg.OtelTest do - use ExUnit.Case, async: false - - alias Hyper.Cfg.Otel - alias Hyper.Cfg.Toml - - setup do - Toml.put_cache(%{}) - on_exit(fn -> Toml.reload() end) - :ok - end - - test "config.exs proto/endpoint/headers produce opentelemetry_exporter opts" do - {:ok, opts} = - Otel.exporter_options( - proto: :grpc, - endpoint: "https://otel.example.com:4317", - headers: %{"authorization" => "Bearer xyz"} - ) - - assert opts[:otlp_protocol] == :grpc - assert opts[:otlp_endpoint] == "https://otel.example.com:4317" - assert opts[:otlp_headers] == [{"authorization", "Bearer xyz"}] - end - - test "reads [otel] toml when config.exs is empty; string proto becomes an atom" do - Toml.put_cache(%{ - "otel" => %{"proto" => "http_protobuf", "endpoint" => "http://collector:4318"} - }) - - {:ok, opts} = Otel.exporter_options([]) - assert opts[:otlp_protocol] == :http_protobuf - assert opts[:otlp_endpoint] == "http://collector:4318" - assert opts[:otlp_headers] == [] - end - - test ":none when no endpoint anywhere" do - assert Otel.exporter_options([]) == :none - end -end From 0430419fab8702ff63957805bfb12ce2582ce16c Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 27 Jun 2026 02:02:06 +0000 Subject: [PATCH 47/47] docs(cookbook): fix config.md correctness against code + architecture - work_dir default is /srv/hyper, store default is /layers (were '-') - budget: cpu/disk_bw/net_bw are all $\beta$ (soft) per architecture.md; cpu_max_cap/disk_bw_max_load/net_bw_max_load were mislabelled $\alpha$ - budget: cpu_max_load/cpu_max_cap/*_max_load are floats (load fraction / core count), not [unit] quantities; cpu_max_cap is optional (default nil) - img.db: keys override the repo's compiled-in config, no 'hyper' default - mark jails uid_gid_range required; drop 'large set of flags' overstatement - typos: exclusively, compile-time config.exs --- docs/cookbook/config.md | 63 ++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md index 6e0e5cbb..5521f5ca 100644 --- a/docs/cookbook/config.md +++ b/docs/cookbook/config.md @@ -6,9 +6,9 @@ Configuring `Hyper` is done through four layers, in priority: | File | Description | | ------------------------ | ----------- | -| `/etc/hyper/config.exs` | The `config.exs` file is exlusively used by the unprivileged `hyper` application. The purpose of this file is to allow you to load configuration values at runtime. If you are using a secrets manager, this is the right place to load the secrets. Must be owned by `root` and only writeable by `root`. | +| `/etc/hyper/config.exs` | The `config.exs` file is exclusively used by the unprivileged `hyper` application. The purpose of this file is to allow you to load configuration values at runtime. If you are using a secrets manager, this is the right place to load the secrets. Must be owned by `root` and only writeable by `root`. | | `/etc/hyper/config.toml` | The `/etc/hyper/config.toml` file is used for static configuration. Unlike `config.exs`, it is used by both `Hyper` and `hyper-suidhelper` which means that it can impact the behavior of a process running under `root`. Must be owned by `root` and only writable by `root`. | -| Compile-Time `config.ex` | The compile-time configuration is generally used to fine-tune the performance of Hyper. You likely do not need to edit most of the configuration fields exposed by this file for day-to-day usage, but they are available for you to tweak. | +| Compile-Time `config.exs` | The compile-time configuration is generally used to fine-tune the performance of Hyper. You likely do not need to edit most of the configuration fields exposed by this file for day-to-day usage, but they are available for you to tweak. | | Defaults | `Hyper` has a set of sane defaults for some, but not all config fields. | **Note that not all layers allow all configuration fields to be tweaked.** Read @@ -28,9 +28,9 @@ Note the keys are abbreviated for better layout: ## Root Keys -| Config Key | `config.exs` | `config.toml` | Default | Notes | -| ---------- | ------------ | ------------- | ------- | ----- | -| `work_dir` | - | `work_dir` | - | [Absolute Path](#absolute-path) where `Hyper` creates its working tree. | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | -------------- | ----- | +| `work_dir` | - | `work_dir` | `"/srv/hyper"` | [Absolute Path](#absolute-path) where `Hyper` creates its working tree. | ### `config.toml` @@ -94,7 +94,7 @@ suidhelper = "/usr/local/bin/hyper-suidhelper" | Config Key | `config.exs` | `config.toml` | Default | Notes | | --------------- | ------------ | ---------------- | --------- | ----- | | `cgroup` | - | `.cgroup` | `"hyper"` | Parent cgroup under which each VM's cgroup is nested. Each VM receives its own ephemeral cgroup which lives under the umbrella of this cgroup. | -| `uid_gid_range` | - | `.uid_gid_range` | - | [Range](#range) limiting the UID/GID values given to VMs. Each VM receives its own UID/GID pair, within these bounds. Must not be an existing user/group. | +| `uid_gid_range` | - | `.uid_gid_range` | - | **Required.** [Range](#range) limiting the UID/GID values given to VMs. Each VM receives its own UID/GID pair, within these bounds. Must not be an existing user/group. | ### `config.toml` @@ -185,18 +185,18 @@ headers = { "authorization" = "Bearer YOUR_TOKEN" } ## Budget Configuration Hyper allows you to control the absolute maximal budgets that are available to -all VMs on a particular node. - -| Config Key | `config.exs` | `config.toml` | Default | Notes | -| ------------------ | ------------------- | ------------------- | --------- | ----- | -| `mem_max` | `.mem_max` | `.mem_max` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of RAM usage. This value **must not** exceed available system memory. | -| `disk_max` | `.disk_max` | `.disk_max` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of disk usage. This value **must not** exceed available system disk space. | -| `cpu_max_load` | `.cpu_max_load` | `.cpu_max_load` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of CPU usage. | -| `cpu_max_cap` | `.cpu_max_cap` | `.cpu_max_cap` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of CPU usage. | -| `disk_bw_cap` | `.disk_bw_cap` | `.disk_bw_cap` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of disk bandwidth. | -| `disk_bw_max_load` | `.disk_bw_max_load` | `.disk_bw_max_load` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of disk bandwidth. | -| `net_bw_cap` | `.net_bw_cap` | `.net_bw_cap` | - | [$\beta$ budget](./architecture.md#budgets) [unit](#unit) of net bandwidth. | -| `net_bw_max_load` | `.net_bw_max_load` | `.net_bw_max_load` | - | [$\alpha$ budget](./architecture.md#budgets) [unit](#unit) of net bandwidth. | +all VMs on a particular node. Every field is required except `cpu_max_cap`. + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ------------------ | ------------------- | ------------------- | ------- | ----- | +| `mem_max` | `.mem_max` | `.mem_max` | - | [$\alpha$ (hard) budget](./architecture.md#budgets): total [unit](#unit) of RAM the node offers VMs. Must not exceed available system memory. | +| `disk_max` | `.disk_max` | `.disk_max` | - | [$\alpha$ (hard) budget](./architecture.md#budgets): total [unit](#unit) of disk the node offers VMs. Must not exceed available system disk. | +| `cpu_max_load` | `.cpu_max_load` | `.cpu_max_load` | - | [$\beta$ (soft) budget](./architecture.md#budgets): instantaneous CPU load threshold, a fraction `0.0`–`1.0` of the cap. | +| `cpu_max_cap` | `.cpu_max_cap` | `.cpu_max_cap` | `nil` | [$\beta$ (soft) budget](./architecture.md#budgets): soft CPU capacity in cores (a float). Optional. | +| `disk_bw_cap` | `.disk_bw_cap` | `.disk_bw_cap` | - | [$\beta$ (soft) budget](./architecture.md#budgets): soft disk-bandwidth capacity ([unit](#unit)). | +| `disk_bw_max_load` | `.disk_bw_max_load` | `.disk_bw_max_load` | - | [$\beta$ (soft) budget](./architecture.md#budgets): disk-bandwidth load threshold, a fraction `0.0`–`1.0` of the cap. | +| `net_bw_cap` | `.net_bw_cap` | `.net_bw_cap` | - | [$\beta$ (soft) budget](./architecture.md#budgets): soft network-bandwidth capacity ([unit](#unit)). | +| `net_bw_max_load` | `.net_bw_max_load` | `.net_bw_max_load` | - | [$\beta$ (soft) budget](./architecture.md#budgets): network-bandwidth load threshold, a fraction `0.0`–`1.0` of the cap. | ### `config.exs` @@ -257,12 +257,13 @@ aarch64 = "/opt/hyper/kernels/vmlinux-aarch64" ## Image Configuration -Hyper's image provisioning layer has a large set of configuration flags -enabling you to tweak how you want Hyper to manage images. +Hyper's image provisioning layer stores read-only layers on a configurable +medium. (The device-mapper geometry behind it is fixed at compile time and +rarely needs tweaking.) -| Config Key | `config.exs` | `config.toml` | Default | Notes | -| ---------- | ------------ | ------------- | ------- | ----- | -| `store` | `.store` | `.store` | - | [Absolute Path](#absolute-path) to the [layer storage medium](./architecture.md#storage). | +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | ------------------- | ----- | +| `store` | `.store` | `.store` | `/layers` | [Absolute Path](#absolute-path) to the [layer storage medium](./architecture.md#storage). | ### `config.exs` @@ -284,12 +285,16 @@ Additionally, sub-sections are available. ### Database Configuration -| Config Key | `config.exs` | `config.toml` | Default | Notes | -| ---------- | ------------ | ------------- | --------- | ----- | -| `database` | `.database` | - | `"hyper"` | | -| `username` | `.username` | - | - | | -| `password` | `.password` | - | - | | -| `hostname` | `.hostname` | - | - | | +These are secrets, so they are `config.exs`-only. Any key you set overrides the +image-database repo's compiled-in connection config; keys you omit keep that +built-in value. + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | ------- | ----- | +| `database` | `.database` | - | - | Postgres database name. | +| `username` | `.username` | - | - | Postgres role. | +| `password` | `.password` | - | - | Postgres password — load it from a secrets manager. | +| `hostname` | `.hostname` | - | - | Postgres host. | ### `config.exs`